import time from collections.abc import Callable from itertools import chain from bs4 import BeautifulSoup from bs4.element import Tag from dateutil import parser import demistomock as demisto # noqa: F401 from CommonServerPython import * # noqa: F401 """ CONSTANTS """ MAX_LIMIT = 1000 ADMIN_AUDITS_MAX_LIMIT = 500 DEFAULT_LIMIT = 250 MAX_FETCH = 5000 VENDOR = "CyberArk" PRODUCT = "EPM" XSIAM_EVENT_TYPE = { "policy_audits": "policy audit raw event details", "admin_audits": "set admin audit data", "detailed_events": "detailed raw", } class Config: """Global static configuration for authentication, OAuth and command outputs.""" # Authentication methods AUTH_METHOD_OAUTH = "Idira OAuth" AUTH_METHOD_EPM = "EPM" AUTH_METHOD_SAML = "SAML" # OAuth (CyberArk Identity / Idira ISPSS) constants GRANT_TYPE_CLIENT_CREDENTIALS = "client_credentials" ACCESS_TOKEN = "access_token" EXPIRES_IN = "expires_in" VALID_UNTIL = "valid_until" DEFAULT_TOKEN_TTL_SECONDS = 6 * 60 * 60 CACHE_BUFFER_SECONDS = 60 # Context outputs prefix per event type. The *-get-events commands expose the parsed/normalized # events under these prefixes so operators can verify the normalization while debugging. OUTPUTS_PREFIX = { "policy_audits": "CyberArkEPM.PolicyAudit", "admin_audits": "CyberArkEPM.AdminAudit", "detailed_events": "CyberArkEPM.Event", } """ CLIENT CLASS """ class Client(BaseClient): def __init__( self, base_url, username, password, application_id, authentication_url=None, application_url=None, verify=True, proxy=False, policy_audits_event_type=None, raw_events_event_type=None, auth_method=None, identity_url=None, web_app_id=None, server_url=None, ): super().__init__(base_url, verify=verify, proxy=proxy) self._headers = { "Accept": "application/json", "Content-Type": "application/json", } self.username = username self.password = password self.application_id = application_id self.authentication_url = authentication_url self.application_url = application_url self.identity_url = identity_url self.web_app_id = web_app_id # `server_url` is the EPM server address used only for the Idira OAuth method (e.g. # https://example.epm.cyberark.com). For the EPM/SAML methods the base URL is resolved # from the login response within the code. Already normalized by `normalize_server_url` # at the parameter-parsing layer. self.server_url = server_url # Resolve the authentication method. When `auth_method` is not provided (e.g. instances # created before the parameter existed), fall back to the legacy behavior: SAML when both # SAML URLs are set, otherwise EPM. This keeps existing instances backward compatible. if not auth_method: auth_method = ( Config.AUTH_METHOD_SAML if (self.authentication_url and self.application_url) else Config.AUTH_METHOD_EPM ) self.auth_method = auth_method if self.auth_method == Config.AUTH_METHOD_OAUTH: self.oauth_auth_to_cyber_ark() elif self.auth_method == Config.AUTH_METHOD_SAML: self.saml_auth_to_cyber_ark() else: self.epm_auth_to_cyber_ark() self.policy_audits_event_type = policy_audits_event_type self.raw_events_event_type = raw_events_event_type self._log_configuration() def _log_configuration(self) -> None: """Log the non-sensitive client configuration to aid troubleshooting. Emitted once per execution, since the Client is constructed a single time in `main()`, so this does not add per-request log noise. Only connection-shaping values are logged. Credentials (username, password) and the Authorization header are deliberately excluded so nothing sensitive reaches the logs; the booleans below record whether a credential was supplied, never its value. """ demisto.debug( "[Client] Configuration: " f"auth_method={self.auth_method!r}, " f"base_url={self._base_url!r}, " f"server_url={self.server_url!r}, " f"identity_url={self.identity_url!r}, " f"web_app_id={self.web_app_id!r}, " f"authentication_url={self.authentication_url!r}, " f"application_url={self.application_url!r}, " f"application_id={self.application_id!r}, " f"policy_audits_event_type={self.policy_audits_event_type!r}, " f"raw_events_event_type={self.raw_events_event_type!r}, " f"verify={self._verify}, " f"has_username={bool(self.username)}, " f"has_password={bool(self.password)}" ) def _get_access_token(self, force_refresh: bool = False) -> str: """Get or refresh the OAuth2 access token, with caching in the integration context. Args: force_refresh: When True, ignore any cached token and always request a fresh one. Used to reactively recover from a server-side 401 (e.g. the token was revoked, rotated, or expired earlier than our computed `valid_until` due to clock skew). """ current_timestamp = int(time.time()) cached_context = get_integration_context() or {} cached_token = cached_context.get(Config.ACCESS_TOKEN) cached_valid_until = cached_context.get(Config.VALID_UNTIL) # Return the cached token if it is still valid (unless a forced refresh was requested). if not force_refresh and cached_token and cached_valid_until: try: valid_until_timestamp = int(float(cached_valid_until)) if current_timestamp < valid_until_timestamp: demisto.debug("[Token Cache] Hit! Token is still valid.") return cached_token demisto.debug("[Token Cache] Miss. Token expired.") except (ValueError, TypeError): demisto.debug("[Token Cache] Error parsing cache. Ignoring.") if not self.identity_url or not self.web_app_id: raise DemistoException("Identity URL and Web App ID are required for OAuth authentication.") token_url = f"{self.identity_url.rstrip('/')}/oauth2/token/{self.web_app_id}" demisto.debug(f"[Token Request] Requesting new token from {token_url}") token_data = {"grant_type": Config.GRANT_TYPE_CLIENT_CREDENTIALS} headers = {"Content-Type": "application/x-www-form-urlencoded"} try: # Use `super()._http_request` (not `self._http_request`) so the token request itself # never enters the OAuth 401 retry logic. A 401 here means bad credentials, and it # should surface directly as an auth failure rather than triggering a refresh-and-retry. token_response = super()._http_request( method="POST", full_url=token_url, data=token_data, headers=headers, auth=(self.username, self.password), resp_type="json", ) except DemistoException as error: error_msg = str(error) demisto.debug(f"[Token Request] Traceback: {traceback.format_exc()}") demisto.error(f"[Token Request] Failed: {error_msg}") raise DemistoException(f"Failed to obtain access token: {error_msg}") access_token = token_response.get(Config.ACCESS_TOKEN) if not access_token: raise DemistoException("Failed to obtain access token. Response missing access_token.") token_expires_in = arg_to_number(token_response.get(Config.EXPIRES_IN)) or Config.DEFAULT_TOKEN_TTL_SECONDS token_valid_until = current_timestamp + token_expires_in - Config.CACHE_BUFFER_SECONDS demisto.debug(f"[Token Request] Success. Expires in {token_expires_in}s.") cached_context[Config.ACCESS_TOKEN] = access_token cached_context[Config.VALID_UNTIL] = str(token_valid_until) set_integration_context(cached_context) return access_token def oauth_auth_to_cyber_ark(self, force_refresh: bool = False) -> None: # Reference: CyberArk Identity (Idira ISPSS) OAuth2 client_credentials flow. # For the Idira OAuth method the EPM server address is provided directly via the # `server_url` parameter, so there is no need to discover the tenant URL at runtime. if not self.server_url: raise DemistoException("Server URL is required for Idira OAuth authentication.") access_token = self._get_access_token(force_refresh=force_refresh) # An empty version selects the version-less path, which the tenant resolves to its latest # deployed version. Interpolating an empty segment would produce a double slash # ("/EPM/API//Sets") and 404, so the two shapes are built separately rather than by # formatting "" into the middle of the path. # The version-less path. CyberArk resolves it to the latest API version deployed on the # tenant, which is what we want: a pinned version is only ever a routing token that can go # stale, and when it does the request fails as a bare 404 with nothing to explain it. # Confirmed against a live tenant - `GET /EPM/API/Sets` returns 200. self._base_url = f"{self.server_url}/EPM/API/" demisto.debug(f"[oauth_auth_to_cyber_ark] Using EPM SET API base URL: {self._base_url}") self._headers["Authorization"] = f"Bearer {access_token}" def _refresh_oauth_token(self) -> None: """Force a new OAuth token and update the Authorization header. Clears the cached token/validity so `_get_access_token(force_refresh=True)` cannot return the stale (now-rejected) token, then re-authenticates. Called reactively when the server returns 401 for a token our cache still considered valid (early revocation, rotation, clock skew, or a shorter-than-reported lifetime). """ demisto.debug("[Token Refresh] Server rejected the token (401). Forcing a new token.") cached_context = get_integration_context() or {} cached_context.pop(Config.ACCESS_TOKEN, None) cached_context.pop(Config.VALID_UNTIL, None) set_integration_context(cached_context) # Re-authenticating issues its own HTTP request (the token request). Guard against # that request re-entering the 401 retry logic (which would recurse infinitely if the # token endpoint itself returns 401, e.g. bad credentials). self._is_authenticating = True try: self.oauth_auth_to_cyber_ark(force_refresh=True) finally: self._is_authenticating = False def _http_request(self, *args, **kwargs): """Wrap BaseClient._http_request to transparently recover from an expired OAuth token. For the OAuth auth method, if a data request fails with 401 Unauthorized, force a token refresh once and retry the request a single time. This complements the proactive time-based refresh (the `CACHE_BUFFER_SECONDS` buffer) by covering cases where the token becomes invalid before its computed expiry. Other auth methods keep the default behavior. """ # Only apply the retry logic for OAuth data requests, and never while we are in the # middle of (re)authenticating, to avoid recursive refresh attempts. if self.auth_method != Config.AUTH_METHOD_OAUTH or getattr(self, "_is_authenticating", False): return super()._http_request(*args, **kwargs) try: return super()._http_request(*args, **kwargs) except DemistoException as error: if not self._is_unauthorized_error(error): raise demisto.debug("[Token Refresh] Received 401 on a data request. Refreshing token and retrying once.") self._refresh_oauth_token() return super()._http_request(*args, **kwargs) @staticmethod def _is_unauthorized_error(error: DemistoException) -> bool: """Return True if the given DemistoException represents an HTTP 401 Unauthorized.""" response = getattr(error, "res", None) if response is not None and getattr(response, "status_code", None) == 401: return True # Fallback for cases where the response object is not attached to the exception. return re.search(r"\b401\b", str(error)) is not None def epm_auth_to_cyber_ark(self): # pragma: no cover data = { "Username": self.username, "Password": self.password, "ApplicationID": self.application_id or "CyberArkXSOAR", } result = self._http_request("POST", url_suffix="/EPM/API/Auth/EPM/Logon", json_data=data) if result.get("IsPasswordExpired"): return_error("CyberArk is reporting that the user password is expired. Terminating script.") self._base_url = urljoin(result.get("ManagerURL"), "/EPM/API/") self._headers["Authorization"] = f"basic {result.get('EPMAuthenticationResult')}" def get_session_token(self) -> str: # pragma: no cover # Reference: https://developer.okta.com/docs/reference/api/authn/#primary-authentication data = { "username": self.username, "password": self.password, } result = self._http_request("POST", full_url=self.authentication_url, json_data=data) demisto.debug(f"[Client.get_session_token] result is: {result}") if result.get("status", "") != "SUCCESS": raise DemistoException( f"Retrieving Okta session token returned status: {result.get('status')}," f" Check your Okta credentials and make sure the user is not blocked by a role." ) return result.get("sessionToken") def get_saml_response(self) -> str: # pragma: no cover # Reference: https://devforum.okta.com/t/how-to-get-saml-assertion-through-an-api/24580 full_url = f"{self.application_url}?onetimetoken={self.get_session_token()}" result = self._http_request("POST", full_url=full_url, resp_type="response") soup = BeautifulSoup(result.text, features="html.parser") saml_input = soup.find("input", {"name": "SAMLResponse"}) saml_response = saml_input.get("value") if isinstance(saml_input, Tag) else None if not isinstance(saml_response, str): # Covers: missing input, missing value, or unexpected non-string attribute type. raise DemistoException("SAMLResponse value not found in authentication response.") return saml_response def saml_auth_to_cyber_ark(self): # pragma: no cover # Reference: https://docs.cyberark.com/EPM/Latest/en/Content/WebServices/SAMLAuthentication.htm headers = {"Content-Type": "application/x-www-form-urlencoded"} data = {"SAMLResponse": self.get_saml_response()} result = self._http_request("POST", url_suffix="/SAML/Logon", headers=headers, data=data) if result.get("IsPasswordExpired"): return_error("CyberArk is reporting that the user password is expired. Terminating script.") self._base_url = urljoin(result.get("ManagerURL"), "/EPM/API/") self._headers["Authorization"] = f"basic {result.get('EPMAuthenticationResult')}" def _log_request_url(self, caller: str, url_suffix: str) -> None: """Log the fully-resolved request URL for a data-plane call. The version segment is chosen once, in `oauth_auth_to_cyber_ark`, and then lives inside `self._base_url` where no per-call log ever showed it. That is precisely how the malformed version segment stayed invisible in a customer's debug log for as long as it did. Logging the resolved URL on every data call means the path shape actually used - version-pinned or version-less - is provable from the logs for *every* endpoint, not just for `Sets`. Args: caller: The calling method, used as the log prefix. url_suffix: The suffix appended to the base URL for this call. """ demisto.debug(f"[{caller}] Request URL: {self._base_url}{url_suffix}") def get_set_list(self) -> dict: self._log_request_url("Client.get_set_list", "Sets") result = self._http_request("GET", url_suffix="Sets") sets = result.get("Sets", []) demisto.debug(f"[Client.get_set_list] Retrieved {len(sets)} sets from API") # The tenant's own set names are logged in full because name resolution is an exact string # match: when it fails, the only way to see why is to compare what was configured against # what the tenant actually returned, character for character. demisto.debug(f"[Client.get_set_list] Set names returned by the tenant: {[entry.get('Name') for entry in sets]}") return result def get_admin_audits(self, set_id: str, from_date: str = "", limit: int = ADMIN_AUDITS_MAX_LIMIT) -> dict: url_suffix = f"Sets/{set_id}/AdminAudit?dateFrom={from_date}&limit={min(limit, ADMIN_AUDITS_MAX_LIMIT)}" self._log_request_url("Client.get_admin_audits", url_suffix) return self._http_request("GET", url_suffix=url_suffix) def get_policy_audits(self, set_id: str, from_date: str = "", limit: int = MAX_LIMIT, next_cursor: str = "start") -> dict: url_suffix = f"Sets/{set_id}/policyaudits/search?nextCursor={next_cursor}&limit={min(limit, MAX_LIMIT)}" filter_params = f"arrivalTime GE {from_date}" if self.policy_audits_event_type: filter_params += f' AND eventType IN {",".join(self.policy_audits_event_type)}' self._log_request_url("Client.get_policy_audits", url_suffix) demisto.debug(f"[Client.get_policy_audits] filter={filter_params}") data = assign_params( filter=filter_params, ) return self._http_request("POST", url_suffix=url_suffix, json_data=data) def get_events(self, set_id: str, from_date: str = "", limit: int = MAX_LIMIT, next_cursor: str = "start") -> dict: url_suffix = f"Sets/{set_id}/Events/Search?nextCursor={next_cursor}&limit={min(limit, MAX_LIMIT)}" filter_params = f"arrivalTime GE {from_date}" if self.raw_events_event_type: filter_params += f' AND eventType IN {",".join(self.raw_events_event_type)}' demisto.debug( f"[Client.get_events] set_id={set_id}, from_date={from_date}, limit={limit}, next_cursor={next_cursor}, " f"raw_events_event_type={self.raw_events_event_type}" ) self._log_request_url("Client.get_events", url_suffix) demisto.debug(f"[Client.get_events] filter={filter_params}") data = assign_params( filter=filter_params, ) return self._http_request("POST", url_suffix=url_suffix, json_data=data) """ HELPER FUNCTIONS """ def create_last_run(set_ids: list, from_date: str) -> dict: """ Gets a list of set_ids and a datetime presentation in str. Args: set_ids (Any): A datetime presentation in str or as a datetime object. from_date (bool): either to increase the datetime with a millisecond (useful for next fetch). Returns: (dict) A dict with a set_id as a key and a dict with the event type (admin_audits, policy_audits, detailed_events) as a key and a dict with `from_date` (from which date the get the event) and `next_cursor` (for the next_fetch) for example { '123': { 'admin_audits': {'from_date': '01-02-2023T23:20:50Z'}, 'policy_audits': {'from_date': '01-02-2023T23:20:50Z', 'next_cursor': 'start'}, 'detailed_events': {'from_date': '01-02-2023T23:20:50Z', 'next_cursor': 'start'}, } '456': { 'admin_audits': {'from_date': '01-02-2023T23:20:50Z'}, 'policy_audits': {'from_date': '01-02-2023T23:20:50Z', 'next_cursor': 'start'}, 'detailed_events': {'from_date': '01-02-2023T23:20:50Z', 'next_cursor': 'start'}, } } """ return { set_id: { "admin_audits": {"from_date": from_date}, "policy_audits": {"from_date": from_date, "next_cursor": "start"}, "detailed_events": {"from_date": from_date, "next_cursor": "start"}, } for set_id in set_ids } def prepare_datetime(date_time: Any, increase: bool = False) -> str: """ Gets a datetime (string or datetime object) and returns a str in ISO format with milliseconds and Z suffix. Args: date_time (Any): A datetime presentation in str or as a datetime object. increase (bool): either to increase the datetime with a millisecond (useful for next fetch). Returns: (str) A datetime presentation in str with milliseconds and Z suffix. 01-02-2023T23:20:50.123Z. """ if isinstance(date_time, str): date_time = parser.parse(date_time, ignoretz=True) if increase: date_time += timedelta(milliseconds=1) date_time_str = date_time.isoformat(timespec="milliseconds") return f"{date_time_str}Z" def prepare_next_run(set_id: str, event_type: str, last_run: dict, last_fetch: dict): # pragma: nocover """ Gets a list of events and adds the `_time` and the `eventTypeXsiam` keys. Args: set_id (str): The set_id that the events are related with. event_type (str): The evnet type, (possible values: policy_audits, detailed_events). last_run (dict): The last run information should be updated with the last fetch information. last_fetch (dict): The last fetch information. Note: Must be called even when zero events are fetched to properly update next_cursor. Failing to update next_cursor may cause infinite fetch loops where stale next_cursor values repeatedly return zero events. """ events = last_fetch.get("events", []) next_cursor = last_fetch.get("next_cursor") # Always update next_cursor to avoid infinite loops last_run[set_id][event_type]["next_cursor"] = next_cursor # Only update from_date when pagination completes (next_cursor == "start") and events from recent fetch exist if events and last_fetch.get("next_cursor") == "start": latest_event = max(events, key=lambda x: parser.parse(x.get("_time"), ignoretz=True)) # type: ignore from_date_next_fetch = prepare_datetime(latest_event.get("_time"), increase=True) # type: ignore last_run[set_id][event_type]["from_date"] = from_date_next_fetch def add_fields_to_events(events: list, date_field: str, event_type: str): """ Gets a list of events and adds the `_time` and the `eventTypeXsiam` keys. Args: events (list): A list of events. date_field (str): The date field from which the _time field is taken. event_type (str): The event type to set in the eventTypeXsiam field. """ for event in events: event["_time"] = event.get(date_field) event["source_log_type"] = XSIAM_EVENT_TYPE.get(event_type) def reconcile_last_run_with_current_sets(last_run: dict, current_set_ids: list, args: dict) -> dict: """ Reconciles the last_run state with the currently configured set IDs. This function handles configuration changes where the user modifies the set names in the integration settings. Without this reconciliation, the integration would continue fetching from old/stale set IDs indefinitely, even after the configuration has been updated to use different sets. Args: last_run (dict): The last run state loaded from demisto.getLastRun() current_set_ids (list): The currently resolved set IDs from get_set_ids_by_set_names() args (dict): Command arguments (used to get from_date for new sets) Returns: dict: Updated last_run with stale sets removed and new sets added """ current_set_ids_set = set(current_set_ids) last_run_set_ids = set(last_run.keys()) if current_set_ids_set != last_run_set_ids: demisto.debug( f"[reconcile_last_run] Set configuration changed! Current: {current_set_ids}, Previous: {list(last_run.keys())}" ) # Remove old sets that are no longer configured # This prevents fetching from sets the user no longer wants to monitor for old_id in last_run_set_ids - current_set_ids_set: del last_run[old_id] demisto.debug(f"[reconcile_last_run] Removed stale set_id from last_run: {old_id}") # Add new sets that were just configured # Initialize them with a fresh from_date to start fetching their events for new_id in current_set_ids_set - last_run_set_ids: from_date = args.get("from_date") or datetime.now() - timedelta(hours=3) last_run[new_id] = create_last_run([new_id], prepare_datetime(from_date))[new_id] demisto.debug(f"[reconcile_last_run] Added new set_id to last_run: {new_id}") demisto.debug(f"[reconcile_last_run] Reconciled last_run now contains set_ids: {list(last_run.keys())}") return last_run def reconcile_split_set_names(configured_names: list[str], tenant_names: list[str]) -> list[str]: """Repair set names that a comma split broke apart, using the tenant's real names as the authority. The *Set name* parameter is a comma-separated list, so a set whose own name contains a comma - such as "CybrWorld-Windows(cyberark software, inc._11)" - arrives here already torn into ["CybrWorld-Windows(cyberark software", "inc._11)"]. Neither fragment matches anything and the fetch fails. On a tenant where EPM has appended an account name like "…, Inc." to every set, that makes the parameter unusable no matter what the operator types. The ambiguity is only unresolvable in isolation: "Alpha, Inc., Beta, Inc." could be two names or four. But we are not in isolation - we know the exact set names the tenant returned, so we can ask which reading corresponds to reality. Consecutive fragments are rejoined and tested against that list, longest run first, so a real name always wins over the fragments it contains. Args: configured_names: The names as parsed from the parameter, possibly split mid-name. tenant_names: The set names the tenant actually returned. Returns: The configured names with any split names rejoined. Fragments that match nothing are preserved unchanged so the caller can still report them as unresolved. """ if not configured_names or not tenant_names: return configured_names tenant_lookup = {name.strip().casefold(): name for name in tenant_names if name} repaired: list[str] = [] index = 0 while index < len(configured_names): # Try the longest run of consecutive fragments first: a shorter run may also match, but a # longer one that matches a real set name is always the better reading. Preferring the # short match would let a set named "Alpha" swallow the first half of "Alpha, Inc.". for end in range(len(configured_names), index, -1): run = configured_names[index:end] # argToList strips whitespace around each element, so the original spacing is gone. # Both spellings are tried because either could be what the operator typed. candidates = [", ".join(run), ",".join(run)] if len(run) > 1 else [run[0]] match = next((tenant_lookup[c.strip().casefold()] for c in candidates if c.strip().casefold() in tenant_lookup), None) if match is not None: if len(run) > 1: demisto.debug( f"[reconcile_split_set_names] Rejoined {len(run)} comma-split fragment(s) into the " f"tenant set name {match!r}." ) repaired.append(match) index = end break else: # No run starting here matches a real set. Keep the fragment as configured so the # caller reports it as unresolved rather than silently dropping it. repaired.append(configured_names[index]) index += 1 if repaired != configured_names: demisto.debug( f"[reconcile_split_set_names] Repaired the configured set names against the tenant list: " f"{configured_names} -> {repaired}" ) return repaired def get_set_ids_by_set_names(client: Client, set_names: list) -> list[str]: """ Gets a list of set names and returns a list of set IDs. Args: client (Client): CyberArkEPM client to use. set_names (list): A list of set names configured in the integration instance. Returns: (dict) A dict of {set_id: events (list events associated with a list of set names)}. """ demisto.debug(f"[get_set_ids_by_set_names] Requested set_names from config: {set_names}") integration_context = get_integration_context() or {} context_set_items = integration_context.get("set_items", {}) demisto.debug(f"[get_set_ids_by_set_names] Cached set_items in context: {context_set_items}") # The cache is keyed by the REPAIRED names, because that is what resolved against the tenant. # A comma-bearing name arrives here as fragments, so comparing the fragments directly against # the cache could never match and every fetch cycle would re-issue GET /Sets forever. Rejoining # the fragments the same way first makes the comparison meaningful again. cached_names = list(context_set_items.keys()) lookup_names = reconcile_split_set_names(set_names, cached_names) if cached_names else set_names if context_set_items.keys() != set(lookup_names): result = client.get_set_list() all_sets = result.get("Sets", []) # Log all available set names from API for debugging all_set_names_from_api = [set_item.get("Name") for set_item in all_sets] demisto.debug(f"[get_set_ids_by_set_names] All available set names from API: {all_set_names_from_api}") # Repair any name the comma split tore apart, now that the tenant's real names are known. # This is the only point in the flow where both halves of the problem are in scope: the # configured value, and the authoritative list to check it against. set_names = reconcile_split_set_names(set_names, all_set_names_from_api) context_set_items = { set_item.get("Name"): set_item.get("Id") for set_item in result.get("Sets", []) if set_item.get("Name") in set_names } # Check for unresolved set names resolved_set_names = set(context_set_items.keys()) unresolved_set_names = set(set_names) - resolved_set_names demisto.debug(f"[get_set_ids_by_set_names] Successfully resolved set names: {resolved_set_names}") demisto.debug(f"[get_set_ids_by_set_names] Resolved set_name -> set_id mapping: {context_set_items}") if unresolved_set_names: # Both sides of the comparison are logged together. Resolution is an exact string # match, so an unresolved name is only ever explicable by seeing it next to the names # the tenant actually returned. Note that a comma-split name would already have been # rejoined above, so anything still unresolved here is a genuine mismatch - a typo, or # a set that no longer exists - rather than a formatting artifact. demisto.error( f"[get_set_ids_by_set_names] Could not resolve the following set names to set IDs: " f"{unresolved_set_names}. These sets will not be fetched. " f"Names available on the tenant: {all_set_names_from_api}. " f"Check for a typo, or for a set that has been renamed or deleted in CyberArk EPM." ) # Merge into the existing context instead of overwriting it, so we don't clobber # other cached keys (e.g. the OAuth `access_token`/`valid_until` written by # `_get_access_token`). integration_context["set_items"] = context_set_items set_integration_context(integration_context) else: demisto.debug(f"[get_set_ids_by_set_names] Using cached set_items from integration context: {lookup_names}") set_ids = list(context_set_items.values()) demisto.debug(f"[get_set_ids_by_set_names] Final set_ids to fetch events from: {set_ids}") return set_ids def get_admin_audits(client: Client, last_run_per_id: dict, limit: int) -> dict[str, list]: # pragma: nocover """ Args: client (Client): CyberArkEPM client to use. last_run_per_id (dict): A dict of set_ids and dates form where to get the events. {'123': '01-02-2023T23:20:50Z'}. limit (int): The maximum events to get. Returns: (dict) A dict of {set_id: events (list events associated with a list of set names)}. """ admin_audits = {} for set_id, last_run in last_run_per_id.items(): from_date = last_run.get("admin_audits", {}).get("from_date") result = client.get_admin_audits(set_id, from_date, limit) admin_audits[set_id] = result.get("AdminAudits", []) total_events = arg_to_number(result.get("TotalCount", 0)) while len(admin_audits[set_id]) < total_events and len(admin_audits[set_id]) < limit: # type: ignore latest_event_date = admin_audits[set_id][-1].get("EventTime") result = client.get_admin_audits(set_id, prepare_datetime(latest_event_date, increase=True), limit) admin_audits[set_id].extend(result.get("AdminAudits", [])) add_fields_to_events(admin_audits[set_id], "EventTime", "admin_audits") return admin_audits def get_events(client_function: Callable, event_type: str, last_run_per_id: dict, limit: int) -> dict[str, dict[str, str | list]]: """ Args: client_function (callable): CyberArkEPM client function to use to get the events. event_type (str): The events type to fetch. last_run_per_id (dict): A dict of set_ids and a dict of dates and next_cursor from where to get the events. {'123': {'from_date': '01-02-2023T23:20:50Z', 'next_cursor': '123465'}}. limit (int): The maximum events to get. Returns: (dict) A dict of {'set_id': {'events' [list events associated with a list of set names], 'next_cursor': '123456'}}. """ demisto.debug( f"[get_events] called with event_type={event_type}, limit={limit}, last_run_set_ids={list(last_run_per_id.keys())}" ) events: dict[str, dict[str, str | list]] = {} for set_id, last_run in last_run_per_id.items(): events[set_id] = {} from_date = last_run.get(event_type).get("from_date") next_cursor = last_run.get(event_type).get("next_cursor") demisto.debug(f"[get_events] requesting first page - {set_id=}, {from_date=}, {next_cursor=}") results = client_function(set_id, from_date, limit, next_cursor) demisto.debug(f"[get_events] set_id={set_id} received {len(results.get('events', []))} events") demisto.debug(f"[get_events] set_id={set_id} nextCursor={results.get('nextCursor')}") events[set_id]["events"] = results.get("events", []) while (next_cursor := results.get("nextCursor")) and len(events[set_id]["events"]) < limit: demisto.debug( f"[get_events] {set_id=} paginating with next_cursor={next_cursor}, current_count=" f"{len(events[set_id]['events'])}" ) results = client_function(set_id, from_date, limit, next_cursor) events[set_id]["events"].extend(results.get("events", [])) # type: ignore demisto.debug(f"[get_events] {set_id=} page received {len(results.get('events', []))}") demisto.debug(f"[get_events] {set_id=} total_count={len(events[set_id]['events'])}") add_fields_to_events(events[set_id]["events"], "arrivalTime", event_type) # type: ignore events[set_id]["next_cursor"] = next_cursor or "start" demisto.debug( f"[get_events] {set_id=}, final_count={len(events[set_id]['events'])}, " f"next_cursor_for_next_fetch={events[set_id]['next_cursor']}" ) return events """ COMMAND FUNCTIONS """ def get_events_command(client: Client, event_type: str, last_run: dict, limit: int) -> tuple[list, CommandResults]: demisto.debug(f"[get_events_command] called with {event_type=}, {limit=}, set_ids={list(last_run.keys())}") if event_type == "admin_audits": results = get_admin_audits(client, last_run, limit) # type: ignore events_list = list(chain(*results.values())) else: if event_type == "policy_audits": results = get_events(client.get_policy_audits, "policy_audits", last_run, limit) # type: ignore if event_type == "detailed_events": results = get_events(client.get_events, "detailed_events", last_run, limit) # type: ignore events_list_of_lists = [value.get("events", []) for value in results.values()] # type: ignore events_list = list(chain(*events_list_of_lists)) demisto.debug(f"[get_events_command] event_type={event_type} total_fetched={len(events_list)}") unique_types = list(dict.fromkeys(e.get("eventType") for e in events_list if isinstance(e, dict))) if events_list else [] demisto.debug(f"[get_events_command] unique_event_types fetched during this fetch={unique_types}") human_readable = tableToMarkdown(string_to_table_header(event_type), events_list) # Expose the parsed/normalized events in `outputs` so operators can verify the normalization # while debugging (in addition to the raw response). return events_list, CommandResults( readable_output=human_readable, outputs_prefix=Config.OUTPUTS_PREFIX[event_type], outputs=events_list, raw_response=events_list, ) def fetch_events( client: Client, last_run: dict, max_fetch: int = MAX_FETCH, enable_admin_audits: bool = False ) -> tuple[list, dict]: """Fetches 3 types of events from CyberArkEPM - admin_audits - policy_audits - events Args: client (Client): CyberArkEPM client to use. last_run (dict): The last run information. max_fetch (int): The max events to return per fetch default is 250. enable_admin_audits (bool): Whether to fetch admin audits events. Defaults is False. Return: (list, dict) A list of events to push to XSIAM, A dict with information for next fetch. """ events: list = [] set_ids_to_process = list(last_run.keys()) demisto.debug(f"[fetch_events] Start fetching, {last_run=}") demisto.debug(f"[fetch_events] Set IDs to process: {set_ids_to_process}") demisto.debug(f"[fetch_events] params: {max_fetch=}, {enable_admin_audits=}") # Restate the base URL at the start of every fetch. The client is built once and its base URL # logged once, but a fetch is what runs every cycle - so this is the line that proves, per # cycle, which URL the event calls actually went to. demisto.debug(f"[fetch_events] Using base_url={client._base_url!r}") if enable_admin_audits: for set_id, admin_audits in get_admin_audits(client, last_run, max_fetch).items(): if admin_audits: last_run[set_id]["admin_audits"]["from_date"] = prepare_datetime(admin_audits[-1].get("EventTime"), increase=True) events.extend(admin_audits) demisto.debug( f"[fetch_events][admin_audits] {set_id=} fetched={len(admin_audits)} " f"new_from_date={last_run[set_id]['admin_audits']['from_date']}" ) for set_id, policy_audits_last_run in get_events(client.get_policy_audits, "policy_audits", last_run, max_fetch).items(): prepare_next_run(set_id, "policy_audits", last_run, policy_audits_last_run) if policy_audits := policy_audits_last_run.get("events", []): demisto.debug( f"[fetch_events][policy_audits] {set_id=} fetched={len(policy_audits)} " f"next_cursor={last_run[set_id]['policy_audits'].get('next_cursor')} " f"from_date_next={last_run[set_id]['policy_audits'].get('from_date')}" ) events.extend(policy_audits) for set_id, detailed_events_last_run in get_events(client.get_events, "detailed_events", last_run, max_fetch).items(): prepare_next_run(set_id, "detailed_events", last_run, detailed_events_last_run) if detailed_events := detailed_events_last_run.get("events", []): demisto.debug( f"[fetch_events][detailed_events] set_id={set_id} fetched={len(detailed_events)} " f"next_cursor={last_run[set_id]['detailed_events'].get('next_cursor')} " f"from_date_next={last_run[set_id]['detailed_events'].get('from_date')}" ) events.extend(detailed_events) unique_types = list(dict.fromkeys(e.get("eventType") for e in events if isinstance(e, dict))) demisto.debug(f"[fetch_events] unique_event_types fetched during this fetch={unique_types}") demisto.debug( f"[fetch_events] Sending {len(events)} events to XSIAM. " f"first_event_keys={(list(events[0].keys()) if events else [])} " f"updated_next_run={last_run}" ) return events, last_run def test_module(client: Client, last_run: dict) -> str: """Test API connectivity and authentication by running a small real fetch. A trimmed fetch is used rather than a single probe call because it exercises the whole chain the customer depends on - authentication, set-name resolution, and the event endpoints - so a misconfiguration surfaces here instead of at the first scheduled fetch. Args: client (Client): CyberArkEPM client to use. last_run (dict): The current last-run object, passed through to the test fetch. Returns: str: 'ok' if the test passed. Any failure raises and fails the test. """ demisto.debug(f"[test_module] Starting test fetch with max_fetch=5 using base_url={client._base_url!r}") fetch_events(client=client, last_run=last_run, max_fetch=5) demisto.info(f"[test_module] PASSED: test fetch succeeded against base_url={client._base_url!r}") return "ok" """ MAIN FUNCTION """ def parse_set_names(raw_set_names: Any) -> list[str]: """Parse the *Set name* parameter into a list of set names. The parameter has always been a comma-separated list, which silently breaks for any set whose name contains a comma. That is not an edge case: CyberArk EPM appends the account name to every set on a tenant, so an account registered as "Example Corp, Inc." yields set names such as "ExWorld-Windows(example corp, inc._11)" - and EPM offers no way to rename them. On such a tenant `argToList` turns one set into two fragments that match nothing, and no combination of sets can be configured at all. The repair happens later, in `reconcile_split_set_names`, where the tenant's real set list is in hand and can say which reading of the commas corresponds to reality. Operators therefore keep entering a plain comma-separated list exactly as before, whatever their names contain. A JSON array is also accepted, for the case where the tenant list is unavailable or a name is genuinely ambiguous, since JSON quotes each element and a comma inside quotes is just a character: ["ExWorld-Windows(example corp, inc._11)", "ExWorld-Linux(example corp, inc._11)"] Args: raw_set_names: The raw parameter value, as configured on the instance. Returns: The configured set names. Empty when nothing was configured. """ if isinstance(raw_set_names, list): # already a list (e.g. multi-select), nothing to parse return [str(name).strip() for name in raw_set_names if str(name).strip()] raw = str(raw_set_names or "").strip() if not raw: return [] if raw.startswith("["): try: parsed = json.loads(raw) except json.JSONDecodeError as json_error: # A value that opens with "[" was clearly meant to be JSON, so a silent fall back to the # comma split would hand back fragments and a baffling "set not found" later on. This # also catches a valid array followed by trailing junk, e.g. '["a"], "b"'. raise DemistoException( f'The "Set name" parameter looks like a JSON array but could not be parsed: {json_error}. ' f'Provide a valid JSON array, for example ["Set One", "Set Two"], or a comma-separated ' f"list of names that do not themselves contain commas." ) from json_error # `parsed` is necessarily a list: json.loads only reaches here for input starting with "[". set_names = [str(name).strip() for name in parsed if str(name).strip()] # The names themselves are logged, not just a count. A count cannot answer the only # question that matters on a tenant with comma-bearing names - did each name survive # whole, or was it split? Seeing the parsed list settles that from the log alone. demisto.debug( f"[parse_set_names] mode=json-array: parsed {len(set_names)} set name(s) from a JSON array. " f"Commas within a name are preserved. names={set_names}" ) return set_names set_names = argToList(raw) demisto.debug( f"[parse_set_names] mode=comma-separated: parsed {len(set_names)} set name(s) using the comma split. " f"A name that itself contains a comma is split into fragments here and is rejoined later by " f"reconcile_split_set_names against the tenant's real set list. names={set_names}" ) return set_names def normalize_server_url(server_url: str | None) -> str | None: """Normalize the *Server URL* parameter so it can be joined into a path without duplicate slashes. Args: server_url: The raw parameter value, which may be None, empty, or have a trailing slash. Returns: The trimmed server URL without a trailing slash, or None when no value was provided. The empty/None case is preserved rather than defaulted, because `oauth_auth_to_cyber_ark` reports a missing Server URL as a user-facing error. """ normalized = (server_url or "").strip().rstrip("/") return normalized or None def validate_params( auth_method: str, base_url: str | None, authentication_url: str | None, application_url: str | None, identity_url: str | None, web_app_id: str | None, server_url: str | None, ) -> None: """Validate the authentication-related parameters based on the selected authentication method. Args: auth_method: The selected authentication method (one of Config.AUTH_METHOD_OAUTH, Config.AUTH_METHOD_SAML, Config.AUTH_METHOD_EPM). base_url: The SAML/EPM Logon URL. authentication_url: The SAML Authentication URL. application_url: The SAML Application URL. identity_url: The Idira OAuth Identity URL. web_app_id: The Idira OAuth Web App ID. server_url: The Idira OAuth Server URL. Raises: SystemExit: Via `return_error` when the required parameters for the selected method are missing or invalid. """ demisto.info(f"Authentication method is: {auth_method}") if auth_method == Config.AUTH_METHOD_OAUTH: if not server_url or not identity_url or not web_app_id: return_error("Server URL, Identity URL, and Web App ID are required for Idira OAuth authentication.") if "/oauth2/token" in (identity_url or ""): return_error( "Identity URL must be the bare FQDN (e.g. https://.id.cyberark.cloud) " "without a '/oauth2/token' suffix." ) elif auth_method == Config.AUTH_METHOD_SAML: if not base_url or not authentication_url or not application_url: return_error("SAML/EPM Logon URL, Authentication URL, and Application URL are required for SAML authentication.") else: # AUTH_METHOD_EPM if not base_url: return_error("SAML/EPM Logon URL is required for EPM authentication.") def main(): # pragma: no cover args = demisto.args() params = demisto.params() command = demisto.command() # Parse parameters base_url = params.get("url") application_id = params.get("application_id") authentication_url = params.get("authentication_url") application_url = params.get("application_url") auth_method = params.get("authentication_method") or Config.AUTH_METHOD_EPM identity_url = params.get("identity_url") web_app_id = params.get("web_app_id") server_url = normalize_server_url(params.get("server_url")) username = params.get("credentials").get("identifier") password = params.get("credentials").get("password") validate_params( auth_method=auth_method, base_url=base_url, authentication_url=authentication_url, application_url=application_url, identity_url=identity_url, web_app_id=web_app_id, server_url=server_url, ) set_names = parse_set_names(params.get("set_name")) enable_admin_audits = argToBoolean(params.get("enable_admin_audits", False)) policy_audits_event_type = argToList(params.get("policy_audits_event_type")) raw_events_event_type = argToList(params.get("raw_events_event_type")) verify_certificate = not params.get("insecure", False) proxy = params.get("proxy", False) max_fetch = arg_to_number(args.get("limit") or params.get("max_fetch") or DEFAULT_LIMIT) max_limit = arg_to_number(args.get("limit", 5)) if not 0 < max_fetch <= MAX_FETCH: # type: ignore demisto.debug(f"`max_fetch` is not in the correct value, setting it to {DEFAULT_LIMIT}.") max_fetch = DEFAULT_LIMIT demisto.info(f"Command being called is {command}") demisto.debug( f"[main] parsed params: {enable_admin_audits=}, {max_fetch=}, " f"{max_limit=}, {set_names=}, " f"{policy_audits_event_type=}, {raw_events_event_type=}" ) try: client = Client( base_url=base_url, username=username, password=password, verify=verify_certificate, proxy=proxy, application_id=application_id, authentication_url=authentication_url, application_url=application_url, policy_audits_event_type=policy_audits_event_type, raw_events_event_type=raw_events_event_type, auth_method=auth_method, identity_url=identity_url, web_app_id=web_app_id, server_url=server_url, ) set_ids = get_set_ids_by_set_names(client, set_names) demisto.debug(f"[main] Resolved {len(set_ids)} set ID(s) from {len(set_names)} configured set name(s)") demisto.debug(f"[main] resolved {set_ids=}") if not set_ids: raise DemistoException( f"No set IDs were resolved from configured set names: {set_names}. " f"Please verify that the set names in the integration configuration match " f"the actual set names in CyberArk EPM." ) if command != "fetch-events" or not demisto.getLastRun(): from_date = args.get("from_date") or datetime.now() - timedelta(hours=3) last_run = create_last_run(set_ids, prepare_datetime(from_date)) demisto.debug(f"[main] initializing last_run for set_ids={set_ids} from_date={prepare_datetime(from_date)}") else: last_run = demisto.getLastRun() demisto.debug(f"[main] loaded existing last_run for set_ids={list(last_run.keys())}") # Reconcile last_run with current configuration to handle set name changes last_run = reconcile_last_run_with_current_sets(last_run, set_ids, args) if command == "test-module": # This is the call made when pressing the integration Test button. result = test_module(client, last_run) return_results(result) elif command == "cyberarkepm-get-admin-audits": demisto.debug( f"[main] executing cyberarkepm-get-admin-audits with limit={max_limit}, " f"from_date={args.get('from_date')}, " f"should_push_events={argToBoolean(args.get('should_push_events', False))}" ) events, command_result = get_events_command(client, "admin_audits", last_run, max_limit) # type: ignore if argToBoolean(args.get("should_push_events", False)): send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"[cyberarkepm-get-admin-audits] send_events_to_xsiam: {len(events)} events, {events=}") return_results(command_result) elif command == "cyberarkepm-get-policy-audits": demisto.debug( f"[main] executing cyberarkepm-get-policy-audits with limit={max_limit}, " f"from_date={args.get('from_date')}, " f"should_push_events={argToBoolean(args.get('should_push_events', False))}" ) events, command_result = get_events_command(client, "policy_audits", last_run, max_limit) # type: ignore if argToBoolean(args.get("should_push_events", False)): send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"[cyberarkepm-get-policy-audits] send_events_to_xsiam: {len(events)} events, {events=}") return_results(command_result) elif command == "cyberarkepm-get-events": demisto.debug( f"[main] executing cyberarkepm-get-events with limit={max_limit}, " f"from_date={args.get('from_date')}, " f"raw_events_event_type={raw_events_event_type}, {enable_admin_audits=}, " f"should_push_events={argToBoolean(args.get('should_push_events', False))}" ) events, command_result = get_events_command(client, "detailed_events", last_run, max_limit) # type: ignore command_results = [command_result] # When admin audits are enabled in the instance configuration, also fetch and include # them in this command's results (in addition to the detailed events). Keep the admin # audits as their own CommandResults so their normalized `outputs` are preserved. if enable_admin_audits: admin_events, admin_command_result = get_events_command(client, "admin_audits", last_run, max_limit) # type: ignore demisto.debug(f"[cyberarkepm-get-events] admin audits enabled, fetched {len(admin_events)} admin audit(s)") events = events + admin_events command_results.append(admin_command_result) if argToBoolean(args.get("should_push_events", False)): send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"[cyberarkepm-get-events] send_events_to_xsiam: {len(events)} events, {events=}") return_results(command_results) elif command in "fetch-events": events, next_run = fetch_events(client, last_run, max_fetch, enable_admin_audits) # type: ignore send_events_to_xsiam(events, vendor=VENDOR, product=PRODUCT) demisto.debug(f"[fetch-events] send_events_to_xsiam: {len(events)} events, {events=}") demisto.setLastRun(next_run) except Exception as error: error_msg = f"Failed to execute {command}. Error: {error!s}" demisto.error(f"{error_msg}\n{traceback.format_exc()}") return_error(error_msg) demisto.debug("CyberArkEPMEventCollector integration finished") if __name__ in ("__main__", "__builtin__", "builtins"): main()