import calendar import json import re import time import requests import urllib3 from CommonServerPython import * from CommonServerUserPython import * class EndaceVisionAPIAdapter: """Adapter for EndaceWebSession which allows a simpler interface to the private Vision API. Implements some of the request methods, so the interface in most cases should be the same. Ensures CSRF tokens are sent correctly in requests. """ API_BASE = "/vision2/data" def __init__(self, endace_session): self.endace_session = endace_session def request(self, method, path, **kwargs): headers = {} if method == "POST": csrf_cookie = self.endace_session.requests.cookies.get("vision2_csrf_cookie") if csrf_cookie: headers = {"XSRF-csrf-token": str(csrf_cookie)} try: r = self.endace_session.requests.request( method, self.endace_session.page(f"{self.API_BASE}/{path}"), headers=headers, **kwargs ) r.raise_for_status() except requests.exceptions.HTTPError as err: return err.response.status_code else: return r def get(self, path, **kwargs): return self.request("GET", path, **kwargs) def post(self, path, **kwargs): return self.request("POST", path, **kwargs) def put(self, path, **kwargs): return self.request("PUT", path, **kwargs) def delete(self, path, **kwargs): return self.request("DELETE", path, **kwargs) class EndaceWebSession: LOGIN_PAGE = "/admin/launch?script=rh&template=login" LOGIN_ACTION = "/admin/launch?script=rh&template=login&action=login" LOGOUT_PAGE = "/admin/launch?script=rh&template=logout&action=logout" def __init__(self, app_url=None, username=None, password=None, cert_verify=False): self.app_url = app_url self.username = username self.password = password self.requests = None self.verify = cert_verify def __enter__(self): self.requests = self._create_login_session() return self def __exit__(self, exception_type, exception_value, traceback): self.close() _ = exception_type _ = exception_value _ = traceback def close(self): self.logout() if self.requests: self.requests.close() self.requests = None def page(self, path="/"): return f"{self.app_url}{path}" def logout(self): if self.requests: logout = self.requests.get(self.page(self.LOGOUT_PAGE)) if logout.status_code == 200: return True else: raise Exception(f"logout to {self.app_url} failed") else: return False def _create_login_session(self): """ Creates a requests.Session object with correct cookies setup for an OSm session. """ sess = requests.Session() r = sess.get(self.page(self.LOGIN_PAGE), verify=self.verify) if r.status_code == 200: csrf_token = EndaceWebSession.find_csrf_token_login(r.content) if csrf_token is None: raise Exception("Could not find CSRF token") # Submit login form login_result = sess.post( self.page(self.LOGIN_ACTION), data={ "_csrf": csrf_token, "d_user_id": "user_id", "t_user_id": "string", "c_user_id": "string", "e_user_id": "true", "f_user_id": str(self.username), "f_password": str(self.password), "Login": "Login", }, headers={"Content-type": "application/x-www-form-urlencoded"}, ) if login_result.status_code == 200 and len(sess.cookies) > 0: return sess else: raise Exception("Login failed") else: raise Exception("Login failed") @staticmethod def find_csrf_token_login(page_content=""): return EndaceWebSession.fetch_csrf_token(str(page_content), r"(class=\"csrf-token\").*?\/>") @staticmethod def fetch_csrf_token(input_text, pattern): """ Fetch CSRF Token from the given input html tag by applying the pattern passed :param input_text: :param pattern: :return: """ m = re.search(pattern=pattern, string=input_text) if not m: return None match_input_tag = m.group(0) if match_input_tag: for replace_tag in ["value", "content"]: m = re.search(pattern=replace_tag + '=".*"', string=match_input_tag) if m: csrf_tag = str(m.group(0)) csrf_tag = csrf_tag.replace(replace_tag + "=", "") csrf_tag = csrf_tag.replace('"', "") return csrf_tag.strip() return None class EndaceVisionData: def __init__(self, args=None): self.args = args def build_search_data(self): investigation_data = { "type": "TrafficBreakdownNG", "breakdown_type": "datasource", "datasource_guids": ["tag:rotation-file"], "start_time": int(self.args["start"]) * 1000, "end_time": int(self.args["end"]) * 1000, "filter": self.get_filters(), "order_by": "bytes", "required_aggregates": ["bytes"], "order_direction": "desc", "other_required": True, "points": 10, "auto_pivot": True, "disable_uvision": False, "disable_mvision": False, } return investigation_data def build_archive_data(self): archive_data = { "filename": self.args["archive_filename"], "deduplication": False, "bidirection": False, "datasources": self.get_datasources(), "timerange": self.get_timerange(), "filters": self.get_filters(), "individualSessionData": False, } return archive_data def get_datasources(self): datasources = { "key": "tag:rotation-file", "datasource": { "id": "tag:rotation-file", "type": "tag", "name": "rotation-file", "probeName": "tag", "displayName": "tag:rotation-file", "status": {"inUse": False, "readers": [], "writers": []}, "vision": True, "mplsLevel1": 1, "mplsLevel2": "BOTTOM", "metadataTimerange": { "visionObjectType": "timerange", "visionObjectValue": { "start": {"seconds": self.args["start"], "nanoseconds": 0}, "end": {"seconds": self.args["end"], "nanoseconds": 0}, }, }, "packetTimerange": { "visionObjectType": "timerange", "visionObjectValue": { "start": {"seconds": self.args["start"], "nanoseconds": 0}, "end": {"seconds": self.args["end"], "nanoseconds": 0}, }, }, "datasourceIds": self.args["ids"], }, "missing": False, } return [datasources] def get_timerange(self): timerange = { "visionObjectType": "timerange", "visionObjectValue": { "start": {"seconds": int(self.args["start"]), "nanoseconds": 0}, "end": {"seconds": int(self.args["end"]), "nanoseconds": 0}, }, } return timerange def get_filters(self): filters = { "visionObjectType": "filter", "visionObjectValue": { "active": "basic", "basic": {"visionObjectType": "basicFilter", "visionObjectValue": {"filters": self.get_basicfilters()}}, }, } return filters def get_basicfilters(self): allfilterslist = [] visionobjecttype = "basicFilterDirectionlessIp" for filtertype in self.args["filterby"]: filterlist = [] if filtertype == 0: visionobjecttype = "basicFilterDirectionlessIp" filterlist.append(self.args["ip"]) if filtertype == 1: visionobjecttype = "basicFilterSourceIp" for sip in self.args["src_host_list"]: filterlist.append(sip) if filtertype == 2: visionobjecttype = "basicFilterDestinationIp" for dip in self.args["dest_host_list"]: filterlist.append(dip) if filtertype == 3: visionobjecttype = "basicFilterSourcePort" for sport in self.args["src_port_list"]: filterlist.append(sport) if filtertype == 4: visionobjecttype = "basicFilterDestinationPort" for dport in self.args["dest_port_list"]: filterlist.append(dport) if filtertype == 5: visionobjecttype = "basicFilterIpProtocol" filterlist.append(self.args["protocol"]) if filtertype == 6: visionobjecttype = "basicFilterDirectionlessPort" filterlist.append(self.args["port"]) if filtertype == 7: visionobjecttype = "basicFilterVlan" for vlan in self.args.vlan1list: filterlist.append(vlan) visionfilter = { "visionObjectType": visionobjecttype, "visionObjectValue": {"version": 1, "include": True, "value": filterlist}, } allfilterslist.append(visionfilter) return allfilterslist class EndaceApp: delta_time = 120 wait_time = 5 def __init__(self, *args): self.args = {} self.applianceurl = args[0] self.username = args[1] self.password = args[2] self.cert_verify = args[3] self.hostname = args[4] @staticmethod def endace_get_input_arguments(args=None): timeframe_converter = { "30seconds": 30, "1minute": 60, "5minutes": 300, "10minutes": 600, "30minutes": 1800, "1hour": 3600, "2hours": 7200, "5hours": 18000, "10hours": 36000, "12hours": 43200, "1day": 86400, "3days": 259200, "5days": 432000, "1week": 604800, } function_args = {} # timestamps function_args["start"] = args.get("start") if args.get("start"): # converting ISO time to epoch time function_args["start"] = date_to_timestamp(args.get("start")) / 1000 function_args["end"] = args.get("end") if args.get("end"): # converting ISO time to epoch time function_args["end"] = date_to_timestamp(args.get("end")) / 1000 function_args["timeframe"] = timeframe_converter.get(args.get("timeframe")) # filter params # args.get("directionless IP") function_args["ip"] = args.get("ip") # args.get("directionless PORT") function_args["port"] = args.get("port") # args.get("src_host_list") function_args["src_host_list"] = list(set(argToList(args.get("src_host_list"))))[:10] # args.get("dest_host_list") function_args["dest_host_list"] = list(set(argToList(args.get("dest_host_list"))))[:10] # args.get("src_port_list") function_args["src_port_list"] = list(set(argToList(args.get("src_port_list"))))[:10] # args.get("dest_port_list") function_args["dest_port_list"] = list(set(argToList(args.get("dest_port_list"))))[:10] function_args["protocol"] = args.get("protocol") # Doing a sanity check on input function arguments # adding a limit to number of filter items to be passed to 10 max if ( len(function_args["src_host_list"]) + len(function_args["dest_host_list"]) + len(function_args["src_port_list"]) + len(function_args["dest_port_list"]) ) > 10: raise ValueError("Wrong number of filters items - Limit search filters to 10 items") if not function_args["ip"] and not function_args["src_host_list"] and not function_args["dest_host_list"]: raise ValueError("Wrong or missing value - Src and Dest IP arguments") if not function_args["start"] and not function_args["end"] and not function_args["timeframe"]: raise ValueError("Wrong arguments - StartTime, EndTime or TimeFrame is invalid ") elif (not function_args["start"] or not function_args["end"]) and not function_args["timeframe"]: raise ValueError("Wrong arguments - either StartTime or EndTime or Timeframe is invalid ") if function_args["start"] and function_args["end"] and function_args["start"] == function_args["end"]: raise ValueError("Wrong arguments - value of StartTime and EndTime argument - both are same") # Logical options for search time: # 1) start and stop time is not provided: will work like search Last n seconds, n = timeframe # 2) only either start or stop time is provided: Search will be either +/- of timeframe of the given time # 3) Both start and stop time is provided: Search using start and stop time, ignore timeframe if not function_args["start"] and not function_args["end"]: function_args["end"] = int(calendar.timegm(time.gmtime()) - 10) function_args["start"] = int(function_args["end"]) - int(function_args["timeframe"]) elif function_args["start"] and not function_args["end"]: if int(function_args["start"]) > (calendar.timegm(time.gmtime()) - 10): raise ValueError(f'Wrong argument - value of StartTime - {args.get("start")} UTC cannot be in future') function_args["end"] = int(function_args["start"]) + int(function_args["timeframe"]) if int(function_args["end"]) > (calendar.timegm(time.gmtime()) - 10): raise ValueError( "Wrong argument - value of EndTime - adjust timeframe argument such that EndTime is not in future" ) elif not function_args["start"] and function_args["end"]: if int(function_args["end"]) > (calendar.timegm(time.gmtime()) - 10): raise ValueError(f'Wrong argument - value of EndTime - {args.get("end")} UTC cannot be in future') function_args["start"] = int(function_args["end"]) - int(function_args["timeframe"]) # search time boundary check if function_args["end"] < function_args["start"]: raise ValueError("Wrong argument - value of EndTime - cannot be before StartTime") if int(function_args["start"]) > (calendar.timegm(time.gmtime()) - 10): raise ValueError(f'Wrong argument - value of StartTime - {args.get("start")} UTC cannot be in future') if int(function_args["end"]) > (calendar.timegm(time.gmtime()) - 10): raise ValueError(f'Wrong argument - value of EndTime - {args.get("end")} UTC cannot be in future') return function_args @staticmethod def handle_error_notifications(eperror): error_dict = { "common.notAuthorized": "Authorization issue due to incorrect RBAC roles on EndaceProbe", "duration.invalidInput": "Fix Invalid search starttime, endtime or timeframe", "timestamp.invalidInput": "Fix Invalid search starttime, endtime or timeframe", "query.serviceLayerError": "One of the search parameters have invalid syntax", "filter.invalidFilterFormat": "One of the search parameters have invalid syntax", "download.emptyDatasource": "Empty Packet Datasource, this happens when packet data has " "rotated out but metadata is still available due to incorrect " "datastore sizing configuration. Contact support@endace.com for " "any technical assistance on optimal datasource sizing", "FileNotFound": "File not found on EndaceProbe", "SearchTimeOut": "Search query has timed out. Improve search by narrowing search filter " "items - IP addresses, Port or Timeframe. If problem persists " "contact support@endace.com to review EndaceProbe configuration", } raise Exception( error_dict.get(eperror, f"try again. contact support@endace.com and report {eperror} if problem persists") ) # search def create_search_task(self, args=None): """create a search task on Endace Probe Args: dict Returns: Dictionary context data in response to the command execution """ input_args_dict = args input_args_dict.update({"filterby": []}) result = {"Task": "CreateSearchTask", "Status": "Started", "Error": "NoError", "JobID": ""} # Filter order if input_args_dict["ip"]: input_args_dict["filterby"].append(0) if input_args_dict["src_host_list"]: input_args_dict["filterby"].append(1) if input_args_dict["dest_host_list"]: input_args_dict["filterby"].append(2) if input_args_dict["src_port_list"]: input_args_dict["filterby"].append(3) if input_args_dict["dest_port_list"]: input_args_dict["filterby"].append(4) if input_args_dict["protocol"]: input_args_dict["filterby"].append(5) if input_args_dict["port"]: input_args_dict["filterby"].append(6) with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rd = api.get(path) if rd.status_code == 200: path = "queries/" evid = EndaceVisionData(input_args_dict) rp = api.post(path, json=evid.build_search_data()) if rp.status_code == 200: try: response = rp.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) payload = response.get("payload") if meta: meta_error = meta.get("error") if meta_error is not None: if meta_error is not False: result["Status"] = "Failed" result["Error"] = str(meta_error) else: if payload is not None: result["JobID"] = payload else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty payload data from {path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Status"] = "Failed" result["Error"] = f"HTTP {rp.status_code} to /{path}" else: result["Status"] = "Failed" result["Error"] = f"HTTP {rd.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result def get_search_status(self, args=None): """get status of the search task on Endace Probe Args: string - Job ID Returns: Dictionary context data in response to the command execution """ result = { "Task": "GetSearchStatus", "Status": "complete", "Error": "NoError", "JobProgress": "0", "DataSources": [], "TotalBytes": 0, "JobID": args, } matching_data = 0 keys = [] values = [] id_to_key_dict = {} app_dict = {} with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rd = api.get(path) if rd.status_code == 200: path = "queries/" + args progress_status = True query_time = calendar.timegm(time.gmtime()) while progress_status: # progress loop # exit when whichever occurrs before, search timeout or search progress = 100% or search returns an # unknown value current_time = calendar.timegm(time.gmtime()) if current_time - query_time > self.delta_time: progress_status = False result["Status"] = "InProgress" result["Error"] = "SearchTimeOut" else: rj = api.get(path) if rj.status_code == 200: # Check metadata for no error. try: response = rj.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) if meta: meta_error = meta.get("error") if meta_error is not None: if meta_error is not False: progress_status = False result["Status"] = "complete" result["Error"] = str(meta_error) else: # check payload for no error payload = response.get("payload") if payload is not None: progress = payload.get("progress") if progress is not None: result["JobProgress"] = str(progress) # check if the Search Job has finished. # if so, return a data dict back to Demisto # if No, Wait and loop in to run another status check, # until "self.delta_time" has elapsed payload_data = payload.get("data") if payload_data is not None and int(progress) == 100: progress_status = False for data_map_dict in payload_data: id_to_key_dict[data_map_dict["id"]] = data_map_dict["name"] for top_key in payload["top_keys"]: keys.append(id_to_key_dict[top_key]) # Calculate Total matching MBytes for top_value in payload["top_values"]: matching_data = matching_data + int(top_value) values.append(str(top_value)) result["TotalBytes"] = int(matching_data) for index in range(len(keys)): app_dict[keys[index]] = values[index] + " Bytes" result["Status"] = str(payload["state"]) result["DataSources"] = keys else: progress_status = False result["Status"] = "Failed" result["Error"] = f"ServerError - empty payload data from {path}" else: progress_status = False result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: progress_status = False result["Status"] = rj.status_code result["Error"] = f"ServerError - HTTP {rj.status_code} to /{path}" # wait time before next run time.sleep(self.wait_time) else: result["Status"] = "Failed" result["Error"] = f"ServerError - HTTP {rd.status_code} to /{path}" if result["Status"] != "complete": self.handle_error_notifications(result["Error"]) return result def delete_search_task(self, args=None): """delete the search task on Endace Probe Args: string - Job ID Returns: dict with delete status """ result = {"Task": "DeleteSearchTask", "Status": "Deleted", "Error": "NoError", "JobID": args} with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rd = api.get(path) if rd.status_code == 200: path = "queries/" + args dr = api.delete(path) if dr.status_code == 200: try: response = dr.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) if meta: meta_error = meta.get("error") if meta_error is not None and meta_error is not False: result["Status"] = "complete" result["Error"] = str(meta_error) else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - HTTP {rd.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result # archive def create_archive_task(self, args=None): """create an archive task on Endace Probe Args: dict Returns: Dictionary context data in response to the command execution """ input_args_dict = args input_args_dict.update({"archive_filename": (args.get("archive_filename") + "-" + str(calendar.timegm(time.gmtime())))}) input_args_dict.update({"filterby": []}) input_args_dict.update({"ids": ""}) result = { "Task": "CreateArchiveTask", "Status": "Started", "Error": "NoError", "JobID": "", "Start": args.get("start"), "End": args.get("end"), "P2Vurl": "", "FileName": args["archive_filename"], } datasource = self.hostname + ":" + input_args_dict["archive_filename"] start_time_in_ms = str(int(input_args_dict["start"]) * 1000) end_time_in_ms = str(int(input_args_dict["end"]) * 1000) p2v_url = ( f'{self.applianceurl}/vision2/pivotintovision/?datasources={datasource}' f'&title={result["FileName"]}&start={start_time_in_ms}&end={end_time_in_ms}' f'&tools=trafficOverTime_by_app%2Cconversations_by_ipaddress' ) # Endace Filter order if input_args_dict["ip"]: input_args_dict["filterby"].append(0) p2v_url = p2v_url + "&ip=" + input_args_dict["ip"] if input_args_dict["src_host_list"]: input_args_dict["filterby"].append(1) src_ip = "" for ip in input_args_dict["src_host_list"]: src_ip = src_ip + "," + ip src_ip = src_ip[1:] p2v_url = p2v_url + "&sip=" + src_ip if input_args_dict["dest_host_list"]: input_args_dict["filterby"].append(2) dest_ip = "" for ip in input_args_dict["dest_host_list"]: dest_ip = dest_ip + "," + ip dest_ip = dest_ip[1:] p2v_url = p2v_url + "&dip=" + dest_ip if input_args_dict["src_port_list"]: input_args_dict["filterby"].append(3) port = "" for sport in input_args_dict["src_port_list"]: port = port + "," + sport port = port[1:] p2v_url = p2v_url + "&sport=" + port if input_args_dict["dest_port_list"]: input_args_dict["filterby"].append(4) port = "" for dport in input_args_dict["dest_port_list"]: port = port + "," + dport port = port[1:] p2v_url = p2v_url + "&dport=" + port if input_args_dict["protocol"]: input_args_dict["filterby"].append(5) if input_args_dict["port"]: input_args_dict["filterby"].append(6) p2v_url = p2v_url + "&port=" + input_args_dict["port"] evid = EndaceVisionData(input_args_dict) with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: # Extract list of rotationfiles datasources and exclude previously archived files rotfile_ids = [] api = EndaceVisionAPIAdapter(sess) path = "datasources" rd = api.get(path) try: response = rd.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: if rd.status_code == 200: payload = response.get("payload") for rotfile in payload: if rotfile["type"] == "rotation_file_v2": rotfile_ids.append(rotfile["id"]) input_args_dict["ids"] = rotfile_ids path = "archive/" rp = api.post(path, json=evid.build_archive_data()) if rp.status_code == 200: try: response = rp.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) payload = response.get("payload") if meta: meta_error = meta.get("error") if meta_error is not None: if meta_error is not False: result["Status"] = "Failed" result["Error"] = str(meta_error) else: if payload is not None: result["JobID"] = payload result["P2Vurl"] = f"[Endace PivotToVision URL]({p2v_url})" else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty payload data from {path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Status"] = "Failed" result["Error"] = f"HTTP {rd.status_code} to /{path}" else: result["Status"] = "Failed" result["Error"] = f"HTTP {rd.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result def get_archive_status(self, args=None): """get status of the archive task on Endace Probe Args: string - Archived File Name Returns: Dictionary context data in response to the command execution """ result = { "Task": "GetArchiveStatus", "Error": "NoError", "Status": "InProgress", "FileName": args["archive_filename"], "FileSize": 0, } with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" progress_status = True query_time = calendar.timegm(time.gmtime()) while progress_status: # wait time before next run time.sleep(self.wait_time) current_time = calendar.timegm(time.gmtime()) if current_time - query_time > self.delta_time: progress_status = False result["Status"] = "InProgress" rf = api.get(path) if rf.status_code == 200: try: response = rf.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) payload = response.get("payload") if meta: meta_error = meta["error"] if meta_error is not None: if meta_error is not False: progress_status = False result["Status"] = "InProgress" result["Error"] = str(meta_error) else: # progress loop # exit at timeout or archive finished # archive_payload = payload for file in payload: if args["archive_filename"] == file["name"]: result["FileName"] = file["name"] if not file["status"]["inUse"]: # archive finished progress_status = False result["FileSize"] = file["usage"] result["Status"] = "Finished" else: result["Status"] = "InProgress" break else: progress_status = False result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: progress_status = False result["Status"] = rf.status_code result["Error"] = f"ServerError - HTTP {rf.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result def delete_archive_task(self, args=None): """delete the search task on Endace Probe Args: string - Job ID Returns: dict with delete status """ result = {"Task": "DeleteArchiveTask", "Error": "NoError", "Status": "Deleted", "JobID": args} with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rd = api.get(path) if rd.status_code == 200: path = "queries/" + args dr = api.delete(path) if dr.status_code == 200: try: response = dr.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) if meta: meta_error = meta.get("error") if meta_error is not None and meta_error is not False: result["Status"] = "complete" result["Error"] = str(meta_error) else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Error"] = rd.status_code result["Error"] = f"ServerError - HTTP {rd.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result def delete_archived_file(self, args=None): """Delete archived file on Endace Probe Args: string - Archived File Name Returns: Dictionary context data in response to the command execution """ result = { "Task": "DeleteArchivedFile", "Error": "NoError", "Status": "FileNotFound", "FileName": args["archived_filename"], } with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rf = api.get(path) if rf.status_code == 200: try: response = rf.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) payload = response.get("payload") if meta: meta_error = meta["error"] if meta_error is not None: if meta_error is not False: result["Status"] = "FileNotFound" result["Error"] = str(meta_error) else: # Delete archived File for file in payload: if result["FileName"] == file["name"] and len(file["id"]) and file["type"] == "archive_file": # File available to delete archived_file_path = f'files?_={calendar.timegm(time.gmtime())!s}000&files={file["id"]}' df = api.delete(archived_file_path) try: response = df.json() except json.decoder.JSONDecodeError: raise else: meta = response.get("meta", {}) if df.status_code == 200: if meta["error"] is None: result["Status"] = "FileNotFound" result["Error"] = meta["error"] else: result["Status"] = "FileDeleted" else: result["Error"] = f"ServerError - HTTP {rf.status_code} to /{path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - HTTP {rf.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result # download def download_pcap(self, args=None): """download a PCAP file from EndaceProbe Args: dict - FileName and FileSize Returns: Dictionary context data in response to the command execution """ result = { "Task": "DownloadPCAP", "Error": "NoError", "Status": "FileNotFound", "FileName": args["filename"], "FileSize": 0, "FileType": "UnKnown", "FileURL": "UnKnown", "FileUser": "UnKnown", } with EndaceWebSession( app_url=self.applianceurl, username=self.username, password=self.password, cert_verify=self.cert_verify ) as sess: api = EndaceVisionAPIAdapter(sess) path = "files" rf = api.get(path) if rf.status_code == 200: try: response = rf.json() except json.decoder.JSONDecodeError: raise Exception(f"JsonDecodeError - path {path}") else: meta = response.get("meta", {}) payload = response.get("payload") if meta: meta_error = meta["error"] if meta_error is not None: if meta_error is not False: result["Status"] = "FileNotFound" result["Error"] = str(meta_error) else: # Download PCAP File for file in payload: if result["FileName"] == file["name"] and len(file["id"]): file_numerical_part = float(re.findall(r"[\d\.]+", file["usage"])[0]) if "KB" in file["usage"]: filesize = file_numerical_part * 0.001 elif "GB" in file["usage"]: filesize = file_numerical_part * 1000 elif "TB" in file["usage"]: filesize = file_numerical_part * 1000000 else: filesize = file_numerical_part * 1 if filesize <= int(args["filesizelimit"]): result["FileName"] = file["name"] + ".pcap" if not file["status"]["inUse"]: # File available to download pcapfile_url_path = f"files/{file['id']}/stream?format=pcap" d = api.get(pcapfile_url_path) if d.status_code == 200: demisto.results( fileResult( f'{result["FileName"]}', d.content, file_type=entryTypes["entryInfoFile"], ) ) result["FileURL"] = ( f"[Endace PCAP URL]" f"({self.applianceurl}/vision2/data/" f"{pcapfile_url_path})" ) result["FileSize"] = file["usage"] result["Status"] = "DownloadFinished" result["FileType"] = file["type"] result["FileUser"] = file["user"] else: result["Status"] = "FileNotFound" result["Error"] = f"ServerError - HTTP {rf.status_code} to /{path}" else: result["Status"] = "FileInUse" else: result["Status"] = "FileExceedsSizeLimit" else: result["Status"] = "Failed" result["Error"] = f"ServerError - empty meta data from {path}" else: result["Status"] = "Failed" result["Error"] = f"ServerError - HTTP {rf.status_code} to /{path}" if result["Status"] == "Failed": self.handle_error_notifications(result["Error"]) return result """ COMMAND FUNCTION """ # Search Commands def endace_create_search_command(app, args): """create search command function Function Params: app: Endace App Client args: start time: in ISO 8601 format end time: in ISO 8601 format ip: comma separated ips to search for port: comma separated ports to search for src ips: comma separated source ips to search for. Src IPs will be OR'd dest ips: comma separated destination ips to search for. Destination IPs will be OR'd src ports: comma separated source ports to search for. Src Ports will be OR'd dest ports: comma separated destination ports to search for. Destination ports will be OR'd protocol: TCP or UDP protocol value search window: number in second Returns: The Search JOB ID. Raises: ValueError: If input argument is in wrong format. """ if len(args.values()): function_args = app.endace_get_input_arguments(args) # calling search task function of app instance result = app.create_search_task(function_args) # create entry context to return to Demisto output = {"Endace.Search.Task(val.JobID == obj.JobID)": result} table_header = ["Task", "JobID", "Status", "Error"] readable_output = tableToMarkdown("EndaceResult", [result], headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("No arguments were provided to search by, at least one must be provided") def endace_delete_search_task_command(app, args): """ Delete search Job on Endace Probe Function Params: app: Endace App Client args: string - Search JOB id Returns: Null Raises: ValueError: If input argument is in wrong format exception: If delete command fails """ jobid = args.get("jobid") if len(re.findall(r"([0-9a-fA-F]+)", jobid)) == 5: # calling search status function of app instance result = app.delete_search_task(jobid) # create entry context to return to Demisto output = {"Endace.Search.Delete(val.JobID == obj.JobID)": result} table_header = ["Task", "JobID", "Status", "Error"] readable_output = tableToMarkdown("EndaceResult", result, headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("Incorrect JOB ID provided") def endace_get_search_status_command(app, args): """Poll search task on Endace Probe by given Job ID Function Params: app: Endace App Client args: Search JobID Returns: Dictionary context data in response to the command execution Raises: ValueError: If input argument is in wrong format. """ jobid = args.get("jobid") if len(re.findall(r"([0-9a-fA-F]+)", jobid)) == 5: # calling search status function of app instance result = app.get_search_status(jobid) # create entry context to return to Demisto output = {"Endace.Search.Response(val.JobID == obj.JobID)": result} table_header = ["Task", "JobID", "Status", "Error", "JobProgress", "DataSources", "TotalBytes"] readable_output = tableToMarkdown("EndaceResult", result, headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("Wrong JOB ID provided") # Archive Commands def endace_create_archive_command(app, args): """create archive command function Function Params: app: Endace App Client args: start time: in ISO 8601 format end time: in ISO 8601 format ip: comma separated ips to search for port: comma separated ports to search for src ips: comma separated source ips to search for. Src IPs will be OR'd dest ips: comma separated destination ips to search for. Destination IPs will be OR'd src ports: comma separated source ports to search for. Src Ports will be OR'd dest ports: comma separated destination ports to search for. Destination ports will be OR'd protocol: TCP or UDP protocol value search window: number representing seconds archive_filename: string representing archive filename Returns: The Archive JOB ID. Raises: ValueError: If input argument is in wrong format. """ if len(args.values()): # archive file name if re.fullmatch(r"[\w0-9_-]+", args.get("archive_filename")) is None: raise ValueError("Wrong format of archive_filename. text, numbers, underscore or dash is supported") function_args = app.endace_get_input_arguments(args) function_args["archive_filename"] = args.get("archive_filename") # calling archive task function of app instance result = app.create_archive_task(function_args) # create entry context to return to Demisto output = {"Endace.Archive.Task(val.JobID == obj.JobID)": result} table_header = ["Task", "FileName", "P2Vurl", "Status", "Error", "JobID"] readable_output = tableToMarkdown("EndaceResult", [result], headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError( "No arguments were provided to search by, at least one Filter item, " "either start/end time or timeframe is required " ) def endace_delete_archive_task_command(app, args): """ Delete archive Job on Endace Probe Function Params: app: Endace App Client args: Archive JOB id - string Returns: Null Raises: ValueError: If input argument is in wrong format exception: If delete command fails """ jobid = args.get("jobid") if re.fullmatch(r"[0-9a-zA-Z\-]+", jobid) is not None: # calling delete archive task function of app instance result = app.delete_archive_task(jobid) # create entry context to return to Demisto output = {"Endace.Archive.Delete(val.JobID == obj.JobID)": result} table_header = ["Task", "JobID", "Status", "Error"] readable_output = tableToMarkdown("EndaceResult", result, headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("Incorrect JOB ID provided") def endace_get_archive_status_command(app, args): """Poll archive task on Endace Probe by given Job ID Function Params: app: Endace App Client args: archive JobID - string Returns: Dictionary context data in response to the command execution Raises: ValueError: If input argument is in wrong format. """ if len(args.values()): function_args = {} # archive file name if re.fullmatch(r"[\w0-9_-]+", args.get("archive_filename")) is None: raise ValueError("Wrong format of archive_filename. text, numbers, underscore or dash is supported") function_args["archive_filename"] = args.get("archive_filename") # calling app instance result = app.get_archive_status(function_args) # create entry context to return to Demisto output = {"Endace.Archive.Response(val.FileName == obj.FileName)": result} table_header = ["Task", "FileName", "Status", "Error", "FileSize"] readable_output = tableToMarkdown("EndaceResult", result, headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("Archived FileName must be provided") def endace_delete_archived_file_command(app, args): """Delete archived file on Endace Probe Function Params: app: Endace App Client args: string , archive filename Returns: Dictionary context data in response to the command execution Raises: ValueError: If input argument is in wrong format. """ if len(args.values()): function_arg = {} # archive file name function_arg["archived_filename"] = args.get("archived_filename") # archive file name if re.fullmatch(r"[\w0-9_-]+", args.get("archived_filename")) is None: raise ValueError("Wrong format of archived_filename. text, numbers, underscore or dash is supported") # calling archive file delete task function of app instance result = app.delete_archived_file(function_arg) # create entry context to return to Demisto output = {"Endace.ArchivedFile.Delete(val.FileName == obj.FileName)": result} table_header = ["Task", "FileName", "Status", "Error"] readable_output = tableToMarkdown("EndaceResult", result, headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("Archived FileName must be provided") # Download Command def endace_download_pcap_command(app, args): """download pcap file function Function Params: app: Endace App Client args: dict of filename: Name of the file to download filesize: limit download to max filesize. Returns: Dictionary context data in response to the command execution Raises: ValueError: If input argument is in wrong format. """ if len(args.values()): function_args = {"filename": args.get("filename"), "filesizelimit": args.get("filesizelimit")} # Doing a sanity check on input function arguments try: int(function_args["filesizelimit"]) except ValueError: raise ValueError("Filesize Limit value is incorrect, must be an integer 1 or greater") else: if int(function_args["filesizelimit"]) < 1: raise ValueError("Filesize Limit value is incorrect, must be an integer 1 or greater") # calling download PCAP function of app instance result = app.download_pcap(function_args) # create entry context to return to Demisto output = {"Endace.Download.PCAP(val.FileName == obj.FileName)": result} table_header = ["Task", "FileName", "Status", "Error", "FileSize", "FileType", "FileUser", "FileURL"] readable_output = tableToMarkdown("EndaceResult", [result], headers=table_header, removeNull=False) raw_response = result return readable_output, output, raw_response else: raise ValueError("FileName must be provided") # Test Command def endace_test_command(appurl, username, password, insecure): if re.match(r"https://[\w\-\.]+\.\w+", appurl) is None: raise ValueError("Wrong Appliance URL. Make sure URL is in format of https://") with EndaceWebSession(app_url=appurl, username=username, password=password, cert_verify=insecure): return "ok" # Main Function def main(): urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) """ COMMANDS + REQUESTS FUNCTIONS """ command = demisto.command() applianceurl = demisto.params().get("applianceurl") username = demisto.params().get("credentials").get("identifier") password = demisto.params().get("credentials").get("password") insecure = not demisto.params().get("insecure", False) hostname = demisto.params().get("hostname") LOG(f"Command being called is {demisto.command()}") try: handle_proxy() if command == "test-module": """ Returning 'ok' indicates that the the user can login to EndaceProbe successfully with his credentials. Returns: 'ok' if test passed, anything else will fail the test """ demisto.results(endace_test_command(applianceurl, username, password, insecure)) else: app = EndaceApp(applianceurl, username, password, insecure, hostname) """ Command Modules """ if command == "endace-create-search": return_outputs(*endace_create_search_command(app, demisto.args())) elif command == "endace-get-search-status": return_outputs(*endace_get_search_status_command(app, demisto.args())) elif command == "endace-delete-search-task": return_outputs(*endace_delete_search_task_command(app, demisto.args())) elif command == "endace-create-archive": return_outputs(*endace_create_archive_command(app, demisto.args())) elif command == "endace-get-archive-status": return_outputs(*endace_get_archive_status_command(app, demisto.args())) elif command == "endace-delete-archive-task": return_outputs(*endace_delete_archive_task_command(app, demisto.args())) elif command == "endace-delete-archived-file": return_outputs(*endace_delete_archived_file_command(app, demisto.args())) elif command == "endace-download-pcap": return_outputs(*endace_download_pcap_command(app, demisto.args())) except Exception as err: err_msg = f"Error in Endace integration: {err}" return_error(err_msg, error=err) # python2 uses __builtin__ python3 uses builtins if __name__ == "__builtin__" or __name__ == "builtins": main()