import json import re import time from xml.dom.minidom import Document, Node, parseString import demistomock as demisto import requests import urllib3 from CommonServerPython import * from CommonServerUserPython import * # disable insecure warnings urllib3.disable_warnings() USERNAME = demisto.params()["credentials"]["identifier"] PASSWORD = demisto.params()["credentials"]["password"] AUTH = ("super/" + USERNAME, PASSWORD) VERIFY_SSL = not demisto.params().get("unsecure", False) HOST = demisto.params()["host"] QUERY_URL = HOST + "/phoenix/rest/query/" REST_ADDRESS = HOST + "/phoenix/rest/h5" EXTENDED_KEYS = {} # type: dict def load_extended_keys(): global EXTENDED_KEYS if demisto.command() == "fetch-incidents": last_run = demisto.getLastRun() EXTENDED_KEYS = last_run.get("extended_keys", {}) else: integration_context = demisto.getIntegrationContext() EXTENDED_KEYS = integration_context.get("extended_keys", {}) if not EXTENDED_KEYS: session = login() url = REST_ADDRESS + "/eventAttributeType/all" response = session.get(url, verify=VERIFY_SSL, auth=AUTH) EXTENDED_KEYS = {attr["attributeId"]: attr["displayName"] for attr in response.json()} if demisto.command() != "fetch-incidents": demisto.setIntegrationContext({"extended_keys": EXTENDED_KEYS}) def parse_resource_type(resource_type): type_to_url_path = { "Reports": "report", "Rules": "rule", "Networks": "resource/network", "Watch Lists": "rule/wl", "Protocols": "resource/port", "Event Type": "eventType", "Malware IP": "mal/ip", "Malware Domains": "mal/site", "Malware Urls": "mal/url", "Malware Hash": "mal/hash", "Malware Processes": "mal/proc", "Country Groups": "resource/geo", "Default Password": "mal/pwd", "Anonymity Network": "mal/proxy", "User Agents": "mal/agent", "Remediations": "remediation", } return type_to_url_path.get(resource_type, resource_type) @logger def validateSuccessfulResponse(resp, error_text): if resp.status_code != 200: return_error(f"Got response status {resp.status_code} when {error_text}") @logger def login(): session = requests.session() login_url = HOST + "/phoenix/login-html.jsf" response = session.get(login_url, verify=VERIFY_SSL) # get the VIEW_STATE from the xml returned in the UI login page. p = re.compile('(value=".{1046}==")') viewState = p.findall(response.text.encode("utf-8")) # type: ignore[arg-type, call-overload] VIEW_STATE = viewState[0][len('value="') :][:-1] data = { "loginHtml": "loginHtml", "loginHtml:username": USERNAME, "loginHtml:password": PASSWORD, "loginHtml:userDomain": "Empty", "loginHtml:loginBtn": "Log In", "loginHtml:domain": "super", "javax.faces.ViewState": VIEW_STATE, } headers = { "Upgrade-Insecure-Requests": "1", "Content-Type": "application/x-www-form-urlencoded", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8," "application/signed-exchange;v=b3;q=0.9", "Accept-Language": "en-US,en;q=0.9,pt-PT;q=0.8,pt;q=0.7", } response = session.post(login_url, headers=headers, data=data, verify=VERIFY_SSL) # type: ignore return session def clear_incident_command(): args = demisto.args() incident_id = args["incident_id"] reason = args.get("close_reason", "") raw_response = clear_incident(incident_id, reason) return_outputs("Incident cleared successfully.", {}, raw_response) @logger def clear_incident(incident_id, reason): session = login() headers = {"Accept": "application/json, text/plain, */*", "Content-Type": "application/json"} response = session.put( HOST + "/phoenix/rest/h5/incident/clear", params={"ids": [incident_id], "user": USERNAME}, headers=headers, data=reason, verify=VERIFY_SSL, ) validateSuccessfulResponse(response, "triggering events report") return response.text @logger def getEventsByIncident(incident_id, max_results, extended_data, max_wait_time): session = login() # response = session.get(HOST + '/phoenix/rest/h5/report/triggerEvent?rawMsg=' + incident_id) # validateSuccessfulResponse(response, "triggering events report") # # try: # jsonRes = response.json() # queryData = jsonRes[0]['right'] # except (ValueError, KeyError): # return_error("Got wrong response format when triggering events report. " # "Expected a json array but got:\n" + response.text) queryData = { "isReportService": True, "selectClause": "eventSeverityCat,incidentLastSeen,eventName,incidentRptDevName,incidentSrc,incidentTarget," "incidentDetail,incidentStatus,incidentReso,incidentId,eventType,incidentTicketStatus," "bizService,count,incidentClearedTime,incidentTicketUser,incidentNotiRecipients," "incidentClearedReason,incidentComments,eventSeverity,incidentFirstSeen,incidentRptIp," "incidentTicketId,customer,incidentNotiStatus,incidentClearedUser,incidentExtUser," "incidentExtClearedTime,incidentExtResoTime,incidentExtTicketId,incidentExtTicketState," "incidentExtTicketType,incidentViewStatus,rawEventMsg,phIncidentCategory,phSubIncidentCategory," "incidentRptDevStatus", "eventFilters": [ {"name": "Filter_OVERALL_STATUS", "singleConstraint": f"(phEventCategory = 1) AND incidentId = {incident_id}"} ], "hints": "IgnoreTime", } return getEventsByQuery( session, queryData, max_results, extended_data, max_wait_time, "FortiSIEM events for Incident " + incident_id, incident_id=incident_id, ) @logger def getEventsByQuery(session, queryData, max_results, extended_data, max_wait_time, tableTitle, incident_id=None): headers = {"Accept": "application/json, text/plain, */*", "Content-Type": "application/json"} response = session.post(REST_ADDRESS + "/report/run", headers=headers, data=json.dumps(queryData), verify=VERIFY_SSL) validateSuccessfulResponse(response, "running report") data = response.json() data["report"] = queryData data = json.dumps(data) # poll until report progress reaches 100 response = session.post(REST_ADDRESS + "/report/reportProgress", headers=headers, data=data, verify=VERIFY_SSL) # response contain the percentage of the report loading while response.text != "100" and max_wait_time > 0: response = session.post(REST_ADDRESS + "/report/reportProgress", headers=headers, data=data, verify=VERIFY_SSL) max_wait_time = int(max_wait_time) - 1 time.sleep(1) params = { "start": 0, "perPage": max_results, "allData": extended_data, } response = session.post(REST_ADDRESS + "/report/resultByReport", params=params, headers=headers, data=data, verify=VERIFY_SSL) try: res = response.json() eventKeys = res["headerData"]["columnNames"] except (ValueError, KeyError): return_error("Got wrong response format when getting report results. Expected a json object but got:\n" + response.text) # reformat results eventData = [] md = "" for key in res["lightValueObjects"]: cur = { "Event ID": key.get("naturalId", ""), "Incident ID": incident_id, } for i in range(len(eventKeys)): if len(key["data"]) == 0 or key["data"][0] == "No report results found.": md = "No report results found." break else: # noqa: RET508 cur[eventKeys[i]] = key["data"][i] if md != "": # no results were found, not need to loop break cur["ExtendedData"] = {} for extItem in key["extData"]: if EXTENDED_KEYS.get(extItem["left"]) is not None: cur[EXTENDED_KEYS.get(extItem["left"]).replace(" ", "")] = extItem["right"] # type: ignore else: cur["ExtendedData"][extItem["left"]] = extItem["right"] eventData.append(cur) md = tableToMarkdown(tableTitle, eventData, eventKeys) if md == "" else md demisto.results( { "ContentsFormat": formats["json"], "Type": entryTypes["note"], "Contents": res, "ReadableContentsFormat": formats["markdown"], "HumanReadable": md, "EntryContext": {'FortiSIEM.Events(val["Event ID"] && val["Event ID"] == obj["Event ID"])': eventData}, } ) @logger def GetEventQuery(): in_xml = create_query_xml("all", interval="1") url = QUERY_URL + "eventQuery" headers = {"Content-Type": "text/xml"} resp = requests.request("POST", url, headers=headers, data=in_xml, verify=VERIFY_SSL, auth=AUTH) validateSuccessfulResponse(resp, "fetching event query") queryId = resp.text if 'error code="255"' in queryId: return_error("Got error code 255 while getting event query. Make sure the query has valid syntax") return queryId @logger def GetIncidentsByOrg(queryId): # The request will poll until the server completes the query. url = QUERY_URL + "progress/" + queryId resp = requests.request("GET", url, verify=VERIFY_SSL, auth=AUTH) while resp.text != "100": resp = requests.request("GET", url, verify=VERIFY_SSL, auth=AUTH) outXML = [] if resp.text == "100": url = QUERY_URL + "events/" + queryId + "/0/1000" resp = requests.request("GET", url, verify=VERIFY_SSL, auth=AUTH) content = resp.text if content != "": outXML.append(content) # this code is taken directly from their documentation. # get all results (last "page" has less than 1000 records) p = re.compile(r'totalCount="\d+"') mlist = p.findall(content) if mlist and mlist[0] != "": mm = mlist[0].replace('"', "") m = int(mm.split("=")[-1]) num = 0 if m > 1000: num = int(m / 1000) if m % 1000 > 0: num += 1 if num > 0: for i in range(num): url = QUERY_URL + "events/" + queryId + "/" + str(i * 1000 + 1) + "/1000" resp = requests.request("GET", url, verify=VERIFY_SSL, auth=AUTH) content = resp.text if content != "": outXML.append(content) else: sys.exit(0) phCustId = "all" param = dumpXML(outXML, phCustId) return param @logger def create_query_xml( include_value, interval="", single_evt_value="phEventCategory=1", interval_type="Minute", attr_list=None, limit="All" ): doc = Document() reports = doc.createElement("Reports") doc.appendChild(reports) report = doc.createElement("Report") report.setAttribute("id", "") report.setAttribute("group", "report") reports.appendChild(report) name = doc.createElement("Name") report.appendChild(name) doc.createTextNode("All Incidents") custScope = doc.createElement("CustomerScope") custScope.setAttribute("groupByEachCustomer", "true") report.appendChild(custScope) include = doc.createElement("Include") if include_value == "all": include.setAttribute("all", "true") custScope.appendChild(include) else: custScope.appendChild(include) include_text = doc.createTextNode(include_value) include.appendChild(include_text) exclude = doc.createElement("Exclude") custScope.appendChild(exclude) description = doc.createElement("description") report.appendChild(description) select = doc.createElement("SelectClause") select.setAttribute("numEntries", limit) report.appendChild(select) attrList = doc.createElement("AttrList") if attr_list: attr_text = doc.createTextNode(str(attr_list)) attrList.appendChild(attr_text) select.appendChild(attrList) reportInterval = doc.createElement("ReportInterval") report.appendChild(reportInterval) window = doc.createElement("Window") window.setAttribute("unit", interval_type) window.setAttribute("val", interval) reportInterval.appendChild(window) pattern = doc.createElement("PatternClause") pattern.setAttribute("window", "3600") report.appendChild(pattern) subPattern = doc.createElement("SubPattern") subPattern.setAttribute("displayName", "Events") subPattern.setAttribute("name", "Events") pattern.appendChild(subPattern) single = doc.createElement("SingleEvtConstr") subPattern.appendChild(single) single_text = doc.createTextNode(single_evt_value) single.appendChild(single_text) _filter = doc.createElement("RelevantFilterAttr") report.appendChild(_filter) return doc.toxml() @logger def dumpXML(xmlList, phCustId): param = [] for xml in xmlList: doc = parseString(xml.encode("utf-8")) for node in doc.getElementsByTagName("events"): for node1 in node.getElementsByTagName("event"): mapping = {} for node2 in node1.getElementsByTagName("attributes"): for node3 in node2.getElementsByTagName("attribute"): item_name = node3.getAttribute("name") for node4 in node3.childNodes: if node4.nodeType == Node.TEXT_NODE: mapping[item_name] = node4.data if phCustId == "all" or mapping["phCustId"] == phCustId: param.append(mapping) return param @logger def buildQueryString(args): res_list = [] for key in args: if "IpAddr" not in key: res_list.append(f'{key} = "{args[key]}"') else: res_list.append(f"{key} = {args[key]}") return " AND ".join(res_list) @logger def getEventsByFilter(maxResults, extendedData, maxWaitTime, reportWindow, reportWindowUnit): session = login() args = demisto.args() del args["maxResults"] del args["extendedData"] del args["maxWaitTime"] del args["reportWindow"] del args["reportWindowUnit"] query_string = buildQueryString(args) query_data = { "isReportService": True, "selectClause": "phRecvTime,reptDevIpAddr,eventType,eventName,rawEventMsg,destIpAddr", "reportWindow": int(reportWindow), "reportWindowUnit": reportWindowUnit, "timeRangeRelative": True, "eventFilters": [{"groupBy": "", "singleConstraint": query_string}], "custId": 1, } return getEventsByQuery(session, query_data, maxResults, extendedData, maxWaitTime, "FortiSIEM Event Results") def parse_cmdb_list(cmdb_device): device_dict = { "DiscoverMethod": cmdb_device.get("discoverMethod", "N/A"), "Approved": cmdb_device.get("approved", "false"), "CreationMethod": cmdb_device.get("creationMethod", "N/A"), "AccessIp": cmdb_device.get("accessIp", "N/A"), "Name": cmdb_device.get("name", "N/A"), "WinMachineGuid": cmdb_device.get("winMachineGuid", "N/A"), "Unmanaged": cmdb_device.get("unmanaged", "false"), "Version": cmdb_device.get("version", "N/A"), "UpdateMethod": cmdb_device.get("updateMethod", "N/A"), } timestamp = cmdb_device.get("discoverTime", None) if timestamp and timestamp.isdigit(): device_dict["DiscoverTime"] = timestamp_to_datestring(timestamp) elif timestamp: device_dict["DiscoverTime"] = timestamp else: device_dict["DiscoverTime"] = "N/A" device_type = cmdb_device.get("deviceType") if device_type: device_dict["DeviceType"] = "{} {}".format(device_type["model"], device_type["vendor"]) else: device_dict["DeviceType"] = "N/A" return device_dict def get_cmdb_devices_command(): args = demisto.args() device_ip = args.get("device_ip") limit = int(args.get("limit")) raw_response = get_cmdb_devices(device_ip, limit) list_of_devices = list(map(parse_cmdb_list, raw_response)) return_outputs(tableToMarkdown("Devices", list_of_devices), {"FortiSIEM.CmdbDevices": list_of_devices}, raw_response) @logger def get_cmdb_devices(device_ip=None, limit=100): cmdb_url = HOST + "/phoenix/rest/cmdbDeviceInfo/devices" if device_ip: cmdb_url += "?includeIps=" + device_ip response = requests.get(cmdb_url, verify=VERIFY_SSL, auth=AUTH) list_of_devices = json.loads(xml2json(response.text)) if "response" in list_of_devices: return_error(list_of_devices["response"]["error"]["description"]) elif "devices" in list_of_devices: list_of_devices = list_of_devices["devices"]["device"] elif "device" in list_of_devices: list_of_devices = [list_of_devices["device"]] return list_of_devices[:limit] @logger def get_events_by_query(query, report_window="60", interval_type="Minute", limit="20", extended_data="false", max_wait_time=60): session = login() query_data = { "isReportService": True, "selectClause": "phRecvTime,reptDevIpAddr,eventType,eventName,rawEventMsg,destIpAddr", "reportWindow": int(report_window), "reportWindowUnit": interval_type, "timeRangeRelative": True, "eventFilters": [{"groupBy": "", "singleConstraint": query}], "custId": 1, } return getEventsByQuery(session, query_data, limit, extended_data, max_wait_time, "FortiSIEM Event Results") def get_lists_command(): raw_resources = get_lists() resources = [] for r in flatten_resources(raw_resources): resources.append( { "DisplayName": r["displayName"], "NatualID": r["naturalId"], "ID": r["id"], "ResourceType": r["groupType"]["displayName"], "Children": [c["displayName"] for c in r["children"]], } ) return_outputs( tableToMarkdown("Lists:", resources, removeNull=True), {"FortiSIEM.ResourceList(val.ID && val.ID == obj.ID)": resources}, raw_response=raw_resources, ) @logger def get_lists(): session = login() url = REST_ADDRESS + "/group/resource" response = session.get(url, verify=VERIFY_SSL, auth=AUTH) return response.json() def flatten_resources(raw_resources): for r in raw_resources: yield r # possible stackoverflow yield from flatten_resources(r["children"]) def add_item_to_resource_list_command(): args = demisto.args() resource_type = parse_resource_type(args["resource_type"]) group_id = args["group_id"] object_info = args.get("object-info", []) object_info = dict(object_property.strip().split("=", 1) for object_property in object_info.split(",")) raw_response = add_item_to_resource_list(resource_type, group_id, object_info) outputs = {"FortiSIEM.Resource(val.id && val.id == obj.id)": createContext(raw_response, removeNull=True)} return_outputs(tableToMarkdown("Resource was added:", raw_response, removeNull=True), outputs, raw_response) @logger def add_item_to_resource_list(resource_type, group_id, object_info): session = login() url = f"{REST_ADDRESS}/{resource_type}/save" object_info["groupId"] = group_id object_info["active"] = True object_info["sysDefined"] = False response = session.post(url, data=json.dumps(object_info), verify=VERIFY_SSL, auth=AUTH) response = response.json() if response.get("code", 0) == -1: return_error(response["msg"]) return response def remove_item_from_resource_list_command(): args = demisto.args() resource_type = parse_resource_type(args["resource_type"]) deleted_ids = args.get("ids", "").split(",") raw_response = remove_item_from_resource_list(resource_type, deleted_ids) return_outputs(raw_response, {}, raw_response=raw_response) @logger def remove_item_from_resource_list(resource_type, deleted_ids): session = login() url = f"{REST_ADDRESS}/{resource_type}/del" response = session.delete(url, params={"ids": json.dumps(deleted_ids)}, verify=VERIFY_SSL, auth=AUTH) if response.text != '"OK"': return_error(response.text) return f"items with id {deleted_ids} were removed." def get_resource_list_command(): args = demisto.args() resource_type = parse_resource_type(args["resource_type"]) group_id = args["group_id"] raw_response = get_resource_list(resource_type, group_id) headers = raw_response.get("headerData", {}).get("keys", []) ec = [] for element in raw_response.get("lightValueObjects", []): e = dict(zip(headers, element.get("data", []))) e["id"] = element.get("objectId") ec.append(e) outputs = {"FortiSIEM.Resource(val.id && val.id == obj.id)": createContext(ec, removeNull=True)} return_outputs(tableToMarkdown("Resource list:", ec, headerTransform=pascalToSpace, removeNull=True), outputs, raw_response) @logger def get_resource_list(resource_type, group_id): session = login() url = f"{REST_ADDRESS}/{resource_type}/list" params = { "groupId": group_id, "start": 0, "size": 50, } response = session.get(url, params=params, verify=VERIFY_SSL, auth=AUTH) response = response.json() if response.get("code", 0) == -1: return_error(response["msg"]) return response def convert_keys_to_snake_case(d): d = {k.replace("-", "_"): v for k, v in d.items()} return d def test(): try: login() except Exception as e: if isinstance(e, requests.exceptions.SSLError): demisto.results("Not verified certificate") else: demisto.results(str(e)) demisto.results("ok") def fetch_incidents(): query_id = GetEventQuery() res = GetIncidentsByOrg(query_id) known_ids = demisto.getLastRun().get("ids", None) if known_ids is None or not known_ids: known_ids = [] incidents = [] for inc in res: if inc.get("incidentId") not in known_ids: incidents.append({"name": inc.get("eventName", "New FortiSIEM Event"), "rawJSON": json.dumps(inc)}) if len(known_ids) >= 1000: known_ids.pop(0) known_ids.append(inc.get("incidentId")) demisto.setLastRun({"ids": known_ids, "extended_keys": EXTENDED_KEYS}) demisto.incidents(incidents) sys.exit(0) def main(): try: handle_proxy() load_extended_keys() if demisto.command() == "test-module": test() elif demisto.command() == "fetch-incidents": fetch_incidents() elif demisto.command() == "fortisiem-get-events-by-incident": args = demisto.args() getEventsByIncident(args["incID"], args["maxResults"], args["extendedData"], args["maxWaitTime"]) elif demisto.command() == "fortisiem-clear-incident": clear_incident_command() elif demisto.command() == "fortisiem-get-events-by-filter": args = demisto.args() getEventsByFilter( args["maxResults"], args["extendedData"], args["maxWaitTime"], args["reportWindow"], args["reportWindowUnit"] ) elif demisto.command() == "fortisiem-get-events-by-query": args = convert_keys_to_snake_case(demisto.args()) get_events_by_query(**args) elif demisto.command() == "fortisiem-get-cmdb-devices": get_cmdb_devices_command() elif demisto.command() == "fortisiem-get-lists": get_lists_command() elif demisto.command() == "fortisiem-add-item-to-resource-list": add_item_to_resource_list_command() elif demisto.command() == "fortisiem-remove-item-from-resource-list": remove_item_from_resource_list_command() elif demisto.command() == "fortisiem-get-resource-list": get_resource_list_command() except Exception as e: if demisto.command() == "fetch-incidents": LOG(str(e)) LOG.print_log() raise else: return_error(str(e)) # python2 uses __builtin__ python3 uses builtins if __name__ == "__builtin__" or __name__ == "builtins": main()