import itertools from collections import Counter from email.utils import parseaddr import dateutil import demistomock as demisto # noqa: F401 import numpy as np import pandas as pd import pytz import tldextract from CommonServerPython import * # noqa: F401 from nltk import sent_tokenize, word_tokenize from nltk.corpus import stopwords from numpy import dot from numpy.linalg import norm from sklearn.feature_extraction.text import CountVectorizer from CommonServerUserPython import * no_fetch_extract = tldextract.TLDExtract(suffix_list_urls=None, cache_dir=False) # type: ignore[arg-type] utc = pytz.UTC SELF_IN_CONTEXT = False EMAIL_BODY_FIELD = "emailbody" EMAIL_SUBJECT_FIELD = "emailsubject" EMAIL_HTML_FIELD = "emailbodyhtml" FROM_FIELD = "emailfrom" FROM_DOMAIN_FIELD = "fromdomain" PREPROCESSED_EMAIL_BODY = "preprocessedemailbody" PREPROCESSED_EMAIL_SUBJECT = "preprocessedemailsubject" MERGED_TEXT_FIELD = "mereged_text" EMAIL_TO_FIELD = "emailto" EMAIL_CC_FIELD = "emailcc" EMAIL_BCC_FIELD = "emailbcc" RECIPIENTS_COLUMNS = [EMAIL_TO_FIELD, EMAIL_CC_FIELD, EMAIL_BCC_FIELD] MIN_CAMPAIGN_SIZE = int(demisto.args().get("minIncidentsForCampaign", 3)) MIN_UNIQUE_RECIPIENTS = int(demisto.args().get("minUniqueRecipients", 2)) DUPLICATE_SENTENCE_THRESHOLD = 0.95 TO_PLOT_CANVAS = demisto.args().get("plotCanvas", "false") == "true" MAX_INCIDENTS_FOR_CANVAS_PLOTTING = 6 MAX_INDICATORS_FOR_CANVAS_PLOTTING = 10 KEYWORDS = [ "#1", "100%", "access", "accordance", "account", "act", "action", "activate", "ad", "affordable", "amazed", "amazing", "apply", "asap", "asked", "attach", "attached", "attachment", "attachments", "attention", "authorize", "authorizing", "avoid", "bank", "bargain", "billing", "bonus", "boss", "bucks", "bulk", "buy", "can't", "cancel", "candidate", "capacity", "card", "cards", "cash", "casino", "caution", "cents", "certified", "chance", "charges", "claim", "claims", "clearance", "click", "collect", "confidentiality", "confirm", "confirmation", "confirmed", "congratulations", "consideration", "consolidate", "consultation", "contact", "contract", "credentials", "credit", "day", "days", "deadline", "deal", "deals", "dear", "debt", "delivered", "delivery", "deposit", "detected", "dhl", "disabled", "discount", "discounts", "document", "documents", "dollar", "dollars", "dropbox", "drugs", "due", "earn", "earnings", "enlarge", "enlargement", "equity", "erection", "erections", "exclusive", "expire", "expires", "fedex", "fees", "file", "finance", "financial", "fraud", "free", "friend", "from", "funds", "gas", "gift", "gimmick", "giveaway", "great", "growth", "guarantee", "guaranteed", "hack", "hacked", "hacker", "hormone", "hosting", "hours", "hurry", "immediate", "immediately", "important", "income", "increase", "instant", "interest", "investment", "invoice", "kindly", "last", "lender", "lenders", "lifetime", "limited", "loan", "loans", "login", "lose", "loss", "luxury", "market", "marketing", "mass", "mastrubate", "mastrubating", "med", "medications", "medicine", "meds", "member", "membership", "million", "millions", "miracle", "money", "monthly", "months", "mortgage", "newsletter", "notification", "notify", "obligation", "offer", "offers", "oil", "only", "open", "opt", "order", "package", "paid", "parcel", "partners", "password", "passwords", "payment", "payments", "paypal", "payroll", "pdf", "penis", "pennies", "permanently", "pharmacy", "pics", "pictures", "pill", "pills", "porn", "porno", "postal", "potential", "pre-approved", "presently", "preview", "price", "prize", "profit", "promise", "promotion", "purchase", "pure", "qualifies", "qualify", "quote", "rates", "receipt", "record", "recorded", "recording", "refund", "request", "requested", "requires", "reserve", "reserves", "review", "risk", "sales", "satisfactin", "satisfaction", "satisfied", "save", "scam", "security", "sensitive", "sex", "share", "shared", "sharing", "shipment", "shipping", "sir", "spam", "special", "spend", "spending", "started", "starting", "stock", "success", "supplies", "supply", "suspended", "temporarily", "terms", "trader", "trading", "traffic", "transaction", "transfer", "trial", "unlimited", "unsecured", "unsolicited", "unsubscribe", "update", "ups", "urgent", "user", "usps", "valium", "verification", "verify", "viagra", "vicodin", "videos", "vids", "viedo", "virus", "waiting", "wallet", "warranty", "web", "weight", "win", "winner", "winning", "wire", "xanax", ] STATUS_DICT = { 0: "Pending", 1: "Active", 2: "Closed", 3: "Archive", } INVALID_KEY_WARNING = ( "Warning: the fields {fields} was not found in the phishing incidents. Please make sure that " "you've specified the machine-name of the fields. The machine name can be found in the " "settings of the incident field you are trying to search." ) INCIDENTS_CONTEXT_TD = "incidents(obj.id == val.id)" def return_outputs_custom(readable_output, outputs=None, tag=None): demisto.debug(f"Entering return_outputs_custom with {tag=}") return_entry = { "Type": entryTypes["note"], "HumanReadable": readable_output, "ContentsFormat": formats["json"], "Contents": outputs, "EntryContext": outputs, } if tag is not None: return_entry["Tags"] = [f"campaign_{tag}"] demisto.results(return_entry) demisto.debug("Exiting return_outputs_custom") def add_context_key(entry_context): demisto.debug("Entering add_context_key") new_context = {} for k, v in entry_context.items(): new_context["{}.{}".format("EmailCampaign", k)] = v demisto.debug(f"Exiting add_context_key, created {len(new_context)} new keys.") return new_context def get_recipients(row): global RECIPIENTS_COLUMNS return list(itertools.chain(*[row[col] for col in RECIPIENTS_COLUMNS])) def extract_domain(address): global no_fetch_extract if address == "": return "" email_address = parseaddr(address)[1] ext = no_fetch_extract(email_address) return f"{ext.domain}.{ext.suffix}" def extract_domain_from_recipients(row): domains_list = [] for address in row["recipients"]: try: domain = extract_domain(address) except Exception: domain = "" domains_list.append(domain) return domains_list def create_context_for_campaign_details(campaign_found=False, incidents_df=None, additional_context_fields: list = None): demisto.debug(f"Entering create_context_for_campaign_details. campaign_found: {campaign_found}") if not campaign_found: demisto.debug("No campaign found, returning minimal context.") return { "isCampaignFound": campaign_found, } else: demisto.debug("Campaign found, building full context.") incident_id = demisto.incident()["id"] incidents_df["recipients"] = incidents_df.apply(lambda row: get_recipients(row), axis=1) incidents_df["recipientsdomain"] = incidents_df.apply(lambda row: extract_domain_from_recipients(row), axis=1) if "removedfromcampaigns" not in incidents_df.columns.tolist(): incidents_df["removedfromcampaigns"] = pd.NA incidents_df["removedfromcampaigns"] = incidents_df["removedfromcampaigns"].apply(lambda x: [] if pd.isna(x) else x) context_keys = { "id", "similarity", FROM_FIELD, FROM_DOMAIN_FIELD, "recipients", "recipientsdomain", "removedfromcampaigns", } invalid_context_keys = set() if additional_context_fields is not None: for key in additional_context_fields: if key in incidents_df.columns: context_keys.add(key) else: invalid_context_keys.add(key) if invalid_context_keys: demisto.debug(f"Found invalid context keys: {invalid_context_keys}") return_warning(INVALID_KEY_WARNING.format(fields=invalid_context_keys)) incidents_context_df = incidents_df.copy(deep=True) incident_df = incidents_context_df[list(context_keys)] # lgtm [py/hash-unhashable-value] if not SELF_IN_CONTEXT: incident_df = incident_df[incident_df["id"] != incident_id] incident_df = incident_df.rename({FROM_DOMAIN_FIELD: "emailfromdomain"}, axis=1) incidents_context = incident_df.fillna(1).to_dict(orient="records") datetimes: pd.DataFrame = incidents_context_df["created_dt"].dropna() min_datetime = min(datetimes).isoformat() demisto.info("Successfully created campaign details context.") return { "isCampaignFound": campaign_found, "involvedIncidentsCount": len(incidents_context_df) if incidents_context_df is not None else 0, "firstIncidentDate": min_datetime, "fieldsToDisplay": additional_context_fields, INCIDENTS_CONTEXT_TD: incidents_context, } def create_context_for_indicators(indicators_df=None): demisto.debug("Entering create_context_for_indicators.") if indicators_df is None: demisto.debug("No indicators DataFrame provided.") indicators_context = [] else: demisto.debug(f"Creating context for {len(indicators_df)} indicators.") indicators_df = indicators_df.rename({"Value": "value"}, axis=1) indicators_df = indicators_df[["id", "value"]] indicators_context = indicators_df.to_dict(orient="records") return {"indicators": indicators_context} def create_empty_context(): demisto.debug("Entering create_empty_context.") context = create_context_for_campaign_details(campaign_found=False) context = add_context_key(context) return context def is_number_of_incidents_too_low(res, incidents): demisto.debug("Entering is_number_of_incidents_too_low check.") if not res["EntryContext"]["isDuplicateIncidentFound"] or len(incidents) < MIN_CAMPAIGN_SIZE: demisto.info(f"Number of incidents ({len(incidents)}) is less than min ({MIN_CAMPAIGN_SIZE}). Not a campaign.") return_outputs_custom("No possible campaign was detected", create_empty_context()) return True demisto.debug("Number of incidents is sufficient.") return False def is_number_of_unique_recipients_is_too_low(incidents): demisto.debug("Entering is_number_of_unique_recipients_is_too_low check.") unique_recipients = Counter([str(i.get(EMAIL_TO_FIELD, "None")) for i in incidents]) unique_recipients += Counter([str(i[EMAIL_CC_FIELD]) for i in incidents if EMAIL_CC_FIELD in i]) unique_recipients += Counter([str(i[EMAIL_BCC_FIELD]) for i in incidents if EMAIL_BCC_FIELD in i]) missing_recipients = unique_recipients["None"] unique_recipients.pop("None", None) if (len(unique_recipients) < MIN_UNIQUE_RECIPIENTS and missing_recipients == 0) or ( 0 < len(unique_recipients) < MIN_UNIQUE_RECIPIENTS and missing_recipients > 0 ): demisto.info( f"Number of unique recipients ({len(unique_recipients)}) is less than min ({MIN_UNIQUE_RECIPIENTS}). Not a campaign." ) msg = "Similar emails were found, but the number of their unique recipients is too low to consider them as campaign.\n " msg += ( "If you wish to consider similar emails as campaign even with low number of unique recipients, " "please change *minUniqueRecipients* argument's value.\n" ) msg += "Details:\n" msg += f"* Found {len(incidents)} similar incidents.\n" msg += f"* Those incidents have {len(unique_recipients)} unique recipients" msg += " ({}).\n".format(", ".join(unique_recipients)) msg += f"* The minimum number of unique recipients for similar emails as campaign: {MIN_UNIQUE_RECIPIENTS}\n" if missing_recipients > 0: msg += ( f"* Could not find email recipient for {missing_recipients}/{len(incidents)} incidents " "(*Email To* field is empty)" ) return_outputs_custom(msg, create_empty_context()) return True demisto.debug("Number of unique recipients is sufficient.") return False def get_str_representation_top_n_values(values_list, counter_tuples_list, top_n): domains_counter_top = counter_tuples_list[:top_n] if len(counter_tuples_list) > top_n: domains_counter_top += [("Other", len(values_list) - sum(x[1] for x in domains_counter_top))] return ", ".join(f"{domain} ({count})" for domain, count in domains_counter_top) def standardize_recipients_column(df, column): if column not in df.columns: df[column] = [[] for _ in range(len(df))] return df df[column] = df[column].apply(argToList) df[column] = df[column].apply(lambda x: [value.strip() for value in x if isinstance(value, str)]) df[column] = df[column].apply(lambda x: [value for value in x if "@" in value]) return df def calculate_campaign_details_table(incidents_df, fields_to_display): demisto.debug("Entering calculate_campaign_details_table.") global RECIPIENTS_COLUMNS n_incidents = len(incidents_df) similarities = incidents_df["similarity"].dropna().to_list() max_similarity = max(similarities) min_similarity = min(similarities) headers = [] contents = [] headers.append("Details") contents.append(f"Found possible campaign of {n_incidents} similar emails") if max_similarity > min_similarity + 10**-3: headers.append("Similarity range") contents.append(f"{min_similarity * 100:.1f}%-{max_similarity * 100:.1f}%") else: headers.append("Similarity") contents.append(f"{max_similarity * 100:.1f}%") incidents_df["created_dt"] = incidents_df["created"].apply(lambda x: dateutil.parser.parse(x)) # type: ignore datetimes = incidents_df["created_dt"].dropna() # type: ignore min_datetime, max_datetime = min(datetimes), max(datetimes) if (max_datetime - min_datetime).days == 0: headers.append("Date") contents.append(max_datetime.strftime("%B %d, %Y")) else: headers.append("Date range") contents.append("{} - {}".format(min_datetime.strftime("%B %d, %Y"), max_datetime.strftime("%B %d, %Y"))) senders = incidents_df[FROM_FIELD].dropna().replace("", np.nan).tolist() senders_counter = Counter(senders).most_common() # type: ignore senders_domain = incidents_df[FROM_DOMAIN_FIELD].replace("", np.nan).dropna().tolist() domains_counter = Counter(senders_domain).most_common() # type: ignore for column in RECIPIENTS_COLUMNS: incidents_df = standardize_recipients_column(incidents_df, column) recipients = [] for column in RECIPIENTS_COLUMNS: for incidents_recipient in incidents_df[column]: recipients += incidents_recipient recipients_counter = Counter(recipients).most_common() # type: ignore if len(senders_counter) == 1: domain_header = "Sender domain" sender_header = "Sender address" elif len(senders_counter) > 1 and len(domains_counter) == 1: domain_header = "Senders domain" sender_header = "Senders addresses" else: domain_header = "Senders domains" sender_header = "Senders addresses" top_n = 3 domain_value = get_str_representation_top_n_values(senders_domain, domains_counter, top_n) sender_value = get_str_representation_top_n_values(senders, senders_counter, top_n) recipients_value = get_str_representation_top_n_values(recipients, recipients_counter, len(recipients_counter)) headers.append(domain_header) contents.append(domain_value) headers.append(sender_header) contents.append(sender_value) headers.append("Recipients") contents.append(recipients_value) for field in fields_to_display: if field in incidents_df.columns: field_values = get_non_na_empty_values(incidents_df, field) if len(field_values) > 0: if field in RECIPIENTS_COLUMNS: field_values = [item for sublist in field_values for item in sublist] elif any(isinstance(field_value, list) for field_value in field_values): flattened_list = [] for item in field_values: if isinstance(item, list): flattened_list.extend(item) else: flattened_list.append(item) field_values = flattened_list field_values_counter = Counter(field_values).most_common() # type: ignore field_value_str = get_str_representation_top_n_values(field_values, field_values_counter, top_n) headers.append(field) contents.append(field_value_str) hr = tableToMarkdown("Possible Campaign Detected", dict(zip(headers, contents)), headers=headers) demisto.info("Successfully calculated campaign details table (Human Readable).") return hr def get_non_na_empty_values(incidents_df, field): field_values = incidents_df[field].replace("", None).dropna().tolist() field_values = [x for x in field_values if len(str(x).strip()) > 0] return field_values def cosine_sim(a, b): return dot(a, b) / (norm(a) * norm(b)) def summarize_email_body(body, subject, nb_sentences=3, subject_weight=1.5, keywords_weight=1.5): demisto.debug("Entering summarize_email_body.") corpus: list[str] = sent_tokenize(body) cv = CountVectorizer(stop_words=list(stopwords.words("english"))) body_arr = cv.fit_transform(corpus).toarray() subject_arr = cv.transform(sent_tokenize(subject)).toarray() word_list = cv.get_feature_names_out() count_list = body_arr.sum(axis=0) + subject_arr.sum(axis=0) * subject_weight duplicate_sentences = [ i for i, arr in enumerate(body_arr) if any(cosine_sim(arr, arr2) > DUPLICATE_SENTENCE_THRESHOLD for arr2 in body_arr[:i]) ] word_frequency = dict(zip(word_list, count_list)) val = sorted(word_frequency.values()) max_frequency = val[-1] for word in word_frequency: word_frequency[word] = word_frequency[word] / max_frequency for word in KEYWORDS: if word in word_frequency: word_frequency[word] *= keywords_weight sentence_rank = [0] * len(corpus) for i, sent in enumerate(corpus): if i in duplicate_sentences: continue for word in word_tokenize(sent): if word.lower() in word_frequency: sentence_rank[i] += word_frequency[word.lower()] sentence_rank[i] = sentence_rank[i] / len(word_tokenize(sent)) # type: ignore top_sentences_indices: np.ndarray = np.argsort(sentence_rank)[::-1][:nb_sentences].tolist() # type: ignore[assignment] summary = [] for sent_i in sorted(top_sentences_indices): # type: ignore sent = corpus[sent_i].strip().replace("\n", " ") if sent_i == 0 and sent_i + 1 not in top_sentences_indices: sent = sent + " ..." elif sent_i + 1 == len(corpus) and sent_i - 1 not in top_sentences_indices: sent = "... " + sent elif sent_i - 1 not in top_sentences_indices and sent_i + 1 not in top_sentences_indices: sent = "... " + sent + " ..." summary.append(sent) demisto.info("Successfully summarized email body.") return "\n".join(summary) def create_email_summary_hr(incidents_df, fields_to_display): demisto.debug("Entering create_email_summary_hr.") clean_email_subject = incidents_df.iloc[0][PREPROCESSED_EMAIL_SUBJECT] email_summary = "*Subject*: " + clean_email_subject.replace("\n", "") + " |" clean_email_body = incidents_df.iloc[0][PREPROCESSED_EMAIL_BODY] email_summary += "\n*Body*: \n" + summarize_email_body(clean_email_body, clean_email_subject) + " |" for word in KEYWORDS: for cased_word in [word.lower(), word.title(), word.upper()]: email_summary = re.sub(rf"(? str: """ convert the output of tableToMarkdown to be vertical. Args: horizontal_md_table: original tableToMarkdown output Returns: md string with rotated table """ demisto.debug("Entering horizontal_to_vertical_md_table.") lines = horizontal_md_table.split("\n") headers_list = lines[1][1:-1].split("|") # To allow pipes in the values, verify that there is a space before or after the pipe, before splitting. regex = rf"(?<=\s){re.escape('|')}|{re.escape('|')}(?=\s)" content_list = re.split(regex, lines[3][1:-1]) new_table = "\n| | |" new_table += "\n|---|---|" for header, content in zip(headers_list, content_list): new_table += f"\n|**{header}**|{content}|" return new_table def return_campaign_details_entry(incidents_df, fields_to_display): demisto.debug("Entering return_campaign_details_entry.") hr_campaign_details = calculate_campaign_details_table(incidents_df, fields_to_display) context, hr_email_summary = create_email_summary_hr(incidents_df, fields_to_display) hr = "\n".join([hr_campaign_details, hr_email_summary]) vertical_hr_campaign_details = horizontal_to_vertical_md_table(hr_campaign_details) demisto.executeCommand( "setIncident", {"emailcampaignsummary": f"{vertical_hr_campaign_details}", "emailcampaignsnippets": hr_email_summary} ) demisto.info("Successfully set incident fields for campaign summary.") return return_outputs_custom(hr, context, tag="campaign_details") def return_no_mututal_indicators_found_entry(): demisto.debug("Entering return_no_mututal_indicators_found_entry.") hr = "No mutual indicators were found." demisto.executeCommand("setIncident", {"emailcampaignmutualindicators": hr}) return_outputs_custom(hr, add_context_key(create_context_for_indicators()), tag="indicators") demisto.info("No mutual indicators found.") def return_indicator_entry(incidents_df): demisto.debug("Entering return_indicator_entry.") indicators_query = "investigationIDs:({})".format(" ".join(f'"{id_}"' for id_ in incidents_df["id"])) fields = ["id", "indicator_type", "investigationIDs", "investigationsCount", "score", "value"] demisto.debug(f"Querying indicators with: {indicators_query}") search_indicators = IndicatorsSearcher(query=indicators_query, limit=150, size=500, filter_fields=",".join(fields)) indicators = [] for res in search_indicators: indicators.extend(res.get("iocs", [])) indicators_df = pd.DataFrame(data=indicators) if len(indicators_df) == 0: demisto.debug("No indicators found after initial search.") return_no_mututal_indicators_found_entry() return indicators_df indicators_df = indicators_df[indicators_df["relatedIncCount"] < 150] indicators_df["Involved Incidents Count"] = indicators_df["investigationIDs"].apply( lambda x: sum(id_ in x for id_ in incidents_df["id"]) ) indicators_df = indicators_df[indicators_df["Involved Incidents Count"] > 1] if len(indicators_df) == 0: demisto.debug("No indicators found with involved count > 1.") return_no_mututal_indicators_found_entry() return indicators_df indicators_df["Id"] = indicators_df["id"].apply(lambda x: f"[{x}](#/indicator/{x})") indicators_df = indicators_df.sort_values(["score", "Involved Incidents Count"], ascending=False) indicators_df["Reputation"] = indicators_df["score"].apply(scoreToReputation) indicators_df = indicators_df.rename({"value": "Value", "indicator_type": "Type"}, axis=1) indicators_headers = ["Id", "Value", "Type", "Reputation", "Involved Incidents Count"] hr = tableToMarkdown("Mutual Indicators", indicators_df.to_dict(orient="records"), headers=indicators_headers) hr_no_title = "\n".join(hr.split("\n")[1:]) demisto.executeCommand("setIncident", {"emailcampaignmutualindicators": hr_no_title}) # without title demisto.info(f"Found {len(indicators_df)} mutual indicators.") return_outputs_custom(hr, add_context_key(create_context_for_indicators(indicators_df)), tag="indicators") return indicators_df def get_comma_sep_list(value): res = [x.strip() for x in value.split(",")] return [x for x in res if x != ""] def get_reputation(id_, indicators_df): if len(indicators_df) == 0: max_reputation = 0 else: relevant_indicators_df = indicators_df[indicators_df["investigationIDs"].apply(lambda x: id_ in x)] if len(relevant_indicators_df) > 0: max_reputation = max(relevant_indicators_df["score"]) else: max_reputation = 0 return scoreToReputation(max_reputation) def return_involved_incidents_entry(incidents_df, indicators_df, fields_to_display): demisto.debug("Entering return_involved_incidents_entry.") incidents_df["Id"] = incidents_df["id"].apply(lambda x: f"[{x}](#/Details/{x})") incidents_df = incidents_df.sort_values("created", ascending=False).reset_index(drop=True) incidents_df["created_dt"] = incidents_df["created"].apply(lambda x: dateutil.parser.parse(x)) # type: ignore incidents_df["Created"] = incidents_df["created_dt"].apply(lambda x: x.strftime("%B %d, %Y")) incidents_df["similarity"] = incidents_df["similarity"].fillna(1) incidents_df["similarity"] = incidents_df["similarity"].apply(lambda x: f"{x * 100:.1f}%") current_incident_id = demisto.incident()["id"] incidents_df["DBot Score"] = incidents_df["id"].apply(lambda id_: get_reputation(id_, indicators_df)) # add a mark at current incident, at its similarity cell incidents_df["similarity"] = incidents_df.apply( lambda x: "{} (current)".format(x["similarity"]) if x["id"] == current_incident_id else x["similarity"], axis=1 ) incidents_df["status"] = incidents_df["status"].apply(lambda x: STATUS_DICT.get(x, "")) incidents_df = incidents_df.rename( {"name": "Name", FROM_FIELD: "Email From", "similarity": "Similarity to Current Incident", "status": "Status"}, axis=1 ) incidents_headers = ["Id", "Created", "Name", "Status", "Email From", "DBot Score", "Similarity to Current Incident"] if fields_to_display is not None: fields_to_display = [f for f in fields_to_display if f in incidents_df.columns] incidents_df[fields_to_display] = incidents_df[fields_to_display].fillna("") fields_to_display = [f for f in fields_to_display if len(get_non_na_empty_values(incidents_df, f)) > 0] incidents_headers += fields_to_display hr = "\n\n" + tableToMarkdown( "Involved Incidents", incidents_df[incidents_headers].to_dict(orient="records"), headers=incidents_headers ) demisto.info("Successfully created 'Involved Incidents' markdown table.") return_outputs_custom(hr, tag="incidents") def draw_canvas(incidents, indicators): demisto.debug(f"Entering draw_canvas for {len(incidents)} incidents and {len(indicators)} indicators.") incident_ids = {x["id"] for x in incidents} filtered_indicators = [] for indicator in indicators: investigations = indicator.get("investigationIDs", []) mutual_incidents_in_canvas = len(set(investigations).intersection(incident_ids)) if mutual_incidents_in_canvas >= 2: filtered_indicators.append(indicator) try: res = demisto.executeCommand( "DrawRelatedIncidentsCanvas", {"relatedIncidentsIDs": list(incident_ids), "indicators": filtered_indicators, "overrideUserCanvas": "true"}, ) if not is_error(res): demisto.info("Successfully generated canvas.") res[-1]["Tags"] = ["canvas"] else: demisto.debug(f"Error drawing canvas: {get_error(res)}") try: demisto.executeCommand("setIncident", {"emailcampaigncanvas": res[-1].get("HumanReadable", "").strip("#")}) except Exception as e: demisto.debug(f"Could not set emailcampaigncanvas incident field: {e}") demisto.results(res) except Exception as e: demisto.debug(f"Exception in draw_canvas: {e}") def analyze_incidents_campaign(incidents, fields_to_display): global TO_PLOT_CANVAS, MAX_INCIDENTS_FOR_CANVAS_PLOTTING, MAX_INDICATORS_FOR_CANVAS_PLOTTING demisto.debug(f"Entering analyze_incidents_campaign for {len(incidents)} incidents.") incidents_df = pd.DataFrame(incidents) return_campaign_details_entry(incidents_df, fields_to_display) indicators_df = return_indicator_entry(incidents_df) return_involved_incidents_entry(incidents_df, indicators_df, fields_to_display) if TO_PLOT_CANVAS and len(incidents_df) <= MAX_INCIDENTS_FOR_CANVAS_PLOTTING: demisto.debug("TO_PLOT_CANVAS is true and incident count is within limit. Drawing canvas.") draw_canvas(incidents, indicators_df.head(MAX_INDICATORS_FOR_CANVAS_PLOTTING).to_dict(orient="records")) else: demisto.debug( f"Skipping canvas plot. TO_PLOT_CANVAS: {TO_PLOT_CANVAS}, " f"Incidents: {len(incidents_df)} (Max: {MAX_INCIDENTS_FOR_CANVAS_PLOTTING})" ) demisto.info("Campaign analysis complete.") def split_non_content_entries(response: list) -> tuple[dict, list]: """ Args: response: A response list from executeCommand. Return: (dict: The last content entry, list: non content entries) """ demisto.debug("Entering split_non_content_entries.") content_entry = response[0] non_content_entries = [] for res_entry in response: if res_entry.get("Contents"): content_entry = res_entry else: non_content_entries.append(res_entry) demisto.debug(f"Found {len(non_content_entries)} non-content entries and 1 content entry.") return content_entry, non_content_entries def main(): global EMAIL_BODY_FIELD, EMAIL_SUBJECT_FIELD, EMAIL_HTML_FIELD, FROM_FIELD, SELF_IN_CONTEXT demisto.debug("Starting EmailCampaign script.") input_args = demisto.args() demisto.debug(f"Script arguments: {input_args}") EMAIL_BODY_FIELD = input_args.get("emailBody", EMAIL_BODY_FIELD) EMAIL_SUBJECT_FIELD = input_args.get("emailSubject", EMAIL_SUBJECT_FIELD) EMAIL_HTML_FIELD = input_args.get("emailBodyHTML", EMAIL_HTML_FIELD) FROM_FIELD = input_args.get("emailFrom", FROM_FIELD) fields_to_display = input_args.get("fieldsToDisplay") SELF_IN_CONTEXT = argToBoolean(input_args.get("includeSelf", "false")) if fields_to_display is not None: input_args["populateFields"] = fields_to_display fields_to_display = get_comma_sep_list(fields_to_display) else: fields_to_display = [] demisto.debug(f"fields_to_display: {fields_to_display}") demisto.debug("Executing FindDuplicateEmailIncidents command.") res = demisto.executeCommand("FindDuplicateEmailIncidents", input_args) if is_error(res): demisto.debug(f"Error from FindDuplicateEmailIncidents: {get_error(res)}") return_error(get_error(res)) content_entry, non_content_entries = split_non_content_entries(res) incidents = json.loads(content_entry["Contents"]) if incidents: demisto.info(f"FindDuplicateEmailIncidents returned {len(incidents)} incidents.") skip_analysis = is_number_of_incidents_too_low(content_entry, incidents) or is_number_of_unique_recipients_is_too_low( incidents ) if not skip_analysis: demisto.debug("Proceeding with campaign analysis.") analyze_incidents_campaign(incidents, fields_to_display) else: demisto.info("Skipping campaign analysis due to pre-check failures (low incidents or recipients).") else: demisto.info("FindDuplicateEmailIncidents returned no incidents.") if non_content_entries: demisto.debug(f"Returning {len(non_content_entries)} non-content entries.") return_results(non_content_entries) demisto.debug("EmailCampaign script finished.") if __name__ in ["__main__", "__builtin__", "builtins"]: main()