EWSO365

import email
import hashlib
from email import _header_value_parser as parser
from email.policy import SMTP, SMTPUTF8
from io import StringIO
from multiprocessing import Process
from xml.sax import SAXParseException
import chardet
import demistomock as demisto  # noqa: F401
import exchangelib
from CommonServerPython import *  # noqa: F401
from EWSApiModule import *
from exchangelib import (
    DELEGATE,
    OAUTH2,
    Body,
    EWSDateTime,
    EWSTimeZone,
    ExtendedProperty,
    FileAttachment,
    HTMLBody,
    ItemAttachment,
)
from exchangelib.errors import (
    ErrorFolderNotFound,
    ErrorMailboxMoveInProgress,
    ErrorMailboxStoreUnavailable,
    MalformedResponseError,
    RateLimitError,
)
from exchangelib.items import Contact, Message
from requests.exceptions import ConnectionError

# Ignore warnings print to stdout
warnings.filterwarnings("ignore")

""" Constants """
INTEGRATION_NAME = get_integration_name()
APP_NAME = "ms-ews-o365"
FOLDER_ID_LEN = 120
MAX_INCIDENTS_PER_FETCH = 200
FETCH_TIME = demisto.params().get("fetch_time") or "10 minutes"

# move results
MOVED_TO_MAILBOX = "movedToMailbox"
MOVED_TO_FOLDER = "movedToFolder"

# item types
FILE_ATTACHMENT_TYPE = "FileAttachment"
ITEM_ATTACHMENT_TYPE = "ItemAttachment"
ATTACHMENT_TYPE = "attachmentType"

TOIS_PATH = "/root/Top of Information Store/"

# context keys
ATTACHMENT_ORIGINAL_ITEM_ID = "originalItemId"
NEW_ITEM_ID = "newItemId"
MESSAGE_ID = "messageId"
ITEM_ID = "itemId"
MAILBOX = "mailbox"
MAILBOX_ID = "mailboxId"
FOLDER_ID = "id"
TARGET_MAILBOX = "receivedBy"

# context paths
CONTEXT_UPDATE_EWS_ITEM = (
    f"EWS.Items((val.{ITEM_ID} === obj.{ITEM_ID} || "
    f"(val.{MESSAGE_ID} && obj.{MESSAGE_ID} && val.{MESSAGE_ID} === obj.{MESSAGE_ID}))"
    f" && val.{TARGET_MAILBOX} === obj.{TARGET_MAILBOX})"
)

CONTEXT_UPDATE_EWS_ITEM_FOR_ATTACHMENT = f"EWS.Items(val.{ITEM_ID} == obj.{ATTACHMENT_ORIGINAL_ITEM_ID})"
CONTEXT_UPDATE_FOLDER = f"EWS.Folders(val.{FOLDER_ID} == obj.{FOLDER_ID})"

# fetch params
LAST_RUN_TIME = "lastRunTime"
LAST_RUN_IDS = "ids"
LAST_RUN_IDS_DICT_REPRESENTATION = "ids_dict"
LAST_RUN_FOLDER = "folderName"
ERROR_COUNTER = "errorCounter"

# Types of filter
MODIFIED_FILTER = "modified-time"
RECEIVED_FILTER = "received-time"

# headers
ITEMS_RESULTS_HEADERS = [
    "sender",
    "subject",
    "hasAttachments",
    "datetimeReceived",
    "receivedBy",
    "author",
    "toRecipients",
    "textBody",
]

# attachment name param
LEGACY_NAME = argToBoolean(demisto.params().get("legacy_name", False))
UTF_8 = "utf-8"

# If you are modifying this probably also need to modify in other files


def exchangelib_cleanup():  # pragma: no cover
    key_protocols = list(exchangelib.protocol.CachingProtocol._protocol_cache.items())
    try:
        exchangelib.close_connections()
    except Exception as ex:
        demisto.error(f"Error was found in exchangelib cleanup, ignoring: {ex}")
    for key, (protocol, _) in key_protocols:
        try:
            if "thread_pool" in protocol.__dict__:
                demisto.debug(f"terminating thread pool key{key} id: {id(protocol.thread_pool)}")
                protocol.thread_pool.terminate()
                del protocol.__dict__["thread_pool"]
            else:
                demisto.info(f"Thread pool not found (ignoring terminate) in protocol dict: {dir(protocol.__dict__)}")
        except Exception as ex:
            demisto.error(f"Error with thread_pool.terminate, ignoring: {ex}")


""" LOGGING """

log_stream = None
log_handler = None


def start_logging():
    global log_stream
    global log_handler
    logging.raiseExceptions = False
    if log_stream is None:
        log_stream = StringIO()
        log_handler = logging.StreamHandler(stream=log_stream)
        log_handler.setFormatter(logging.Formatter(logging.BASIC_FORMAT))
        logger = logging.getLogger()
        logger.addHandler(log_handler)
        logger.setLevel(logging.DEBUG)


""" Helper Functions """


def get_client_from_params(params: dict) -> EWSClient:
    """
    Parse the integration params and create an EWS client object
    Args:
        params (dict): dict received from demisto.params()

    Returns:
        EWSClient: EWS client object to interact with the exchange API
    """
    client_id = params.get("_client_id") or params.get("client_id", "")
    client_secret = (params.get("credentials") or {}).get("password") or params.get("client_secret", "")
    tenant_id = params.get("_tenant_id") or params.get("tenant_id", "")
    if not client_secret:
        raise Exception("Key / Application Secret must be provided.")
    elif not client_id:
        raise Exception("ID / Application ID must be provided.")
    elif not tenant_id:
        raise Exception("Token / Tenant ID must be provided.")

    access_type = params.get("access_type", DELEGATE) or DELEGATE
    access_type = (access_type[0] if isinstance(access_type, list) else access_type).lower()
    default_target_mailbox = params.get("default_target_mailbox", "")
    max_fetch = min(int(params.get("max_fetch", MAX_INCIDENTS_PER_FETCH)), MAX_INCIDENTS_PER_FETCH)
    azure_cloud = get_azure_cloud(params, INTEGRATION_NAME)
    ews_server = f"{azure_cloud.endpoints.exchange_online}/EWS/Exchange.asmx/"
    folder = params.get("folder", "Inbox")
    is_public_folder = argToBoolean(params.get("is_public_folder", False))
    request_timeout = int(params.get("request_timeout", 120))
    mark_as_read = params.get("mark_as_read", False)
    incident_filter = IncidentFilter(
        params.get("incidentFilter", IncidentFilter.RECEIVED_FILTER) or IncidentFilter.RECEIVED_FILTER
    )
    self_deployed = argToBoolean(params.get("self_deployed", False))
    insecure = argToBoolean(params.get("insecure", False))
    proxy = params.get("proxy", False)

    return EWSClient(
        client_id=client_id,
        client_secret=client_secret,
        access_type=access_type,
        default_target_mailbox=default_target_mailbox,
        max_fetch=max_fetch,
        ews_server=ews_server,
        auth_type=OAUTH2,
        version="O365",
        folder=folder,
        is_public_folder=is_public_folder,
        request_timeout=request_timeout,
        mark_as_read=mark_as_read,
        incident_filter=incident_filter,
        azure_cloud=azure_cloud,
        tenant_id=tenant_id,
        self_deployed=self_deployed,
        app_name=APP_NAME,
        insecure=insecure,
        proxy=proxy,
    )


def get_attachment_name(attachment_name, eml_extension=False, content_id="", is_inline=False):
    """
    Retrieve attachment name or error string if none is provided
    :param attachment_name: attachment name to retrieve
    :param eml_extension: Indicates whether the eml extension should be added
    :return: string
    """
    if is_inline and content_id and content_id != "None" and not LEGACY_NAME:
        if attachment_name is None or attachment_name == "":
            return (
                f"{content_id}-attachmentName-demisto_untitled_attachment.eml"
                if eml_extension
                else f"{content_id}-attachmentName-demisto_untitled_attachment"
            )
        elif eml_extension and not attachment_name.endswith(".eml"):
            return f"{content_id}-attachmentName-{attachment_name}.eml"
        return f"{content_id}-attachmentName-{attachment_name}"
    if attachment_name is None or attachment_name == "":
        return "demisto_untitled_attachment.eml" if eml_extension else "demisto_untitled_attachment"
    elif eml_extension and not attachment_name.endswith(".eml"):
        return f"{attachment_name}.eml"
    return attachment_name


def prepare_args(args: dict[str, str]) -> dict:
    """
    Prepare arguments to be used as the API expects it
    :param args: demisto args
    :return: transformed args
    """
    args = {k.replace("-", "_"): v for k, v in list(args.items())}
    if "is_public" in args:
        args["is_public"] = args["is_public"] == "True"  # type: ignore[assignment]
    if "from" in args:
        args["from_address"] = args.pop("from")
    return args


def get_limited_number_of_messages_from_qs(qs, limit):  # pragma: no cover
    """
    Retrieve a limited number of messages from query search
    :param qs: query search to execute
    :param limit: limit on number of items to retrieve from search
    :return: list of exchangelib.Message
    """
    count = 0
    results = []
    for item in qs:
        if count == limit:
            break
        if isinstance(item, Message):
            count += 1
            results.append(item)
    return results


def keys_to_camel_case(value):  # pragma: no cover
    """
    Transform keys from snake to camel case (does nothing if no snakes are found)
    :param value: value to transform
    :return: transformed value
    """

    def str_to_camel_case(snake_str):
        components = snake_str.split("_")
        return components[0] + "".join(x.title() for x in components[1:])

    if value is None:
        return None
    if isinstance(value, list | set):
        return list(map(keys_to_camel_case, value))
    if isinstance(value, dict):
        return {
            keys_to_camel_case(k): keys_to_camel_case(v) if isinstance(v, list | dict) else v for (k, v) in list(value.items())
        }

    return str_to_camel_case(value)


def get_last_run(client: EWSClient, last_run=None):
    """
    Retrieve the last run time
    :param client: EWS Client
    :param last_run: (Optional) last run object
    :return: last run dict
    """
    if not last_run or last_run.get(LAST_RUN_FOLDER) != client.folder_name:
        last_run = {
            LAST_RUN_TIME: None,
            LAST_RUN_FOLDER: client.folder_name,
            LAST_RUN_IDS_DICT_REPRESENTATION: {},
        }
    if last_run.get(LAST_RUN_TIME):
        last_run[LAST_RUN_TIME] = EWSDateTime.from_string(last_run[LAST_RUN_TIME])

    # In case we have existing last_run data
    if not last_run.get(LAST_RUN_IDS_DICT_REPRESENTATION):
        if not last_run.get(LAST_RUN_IDS):
            last_run[LAST_RUN_IDS_DICT_REPRESENTATION] = {}
        else:
            last_run[LAST_RUN_IDS_DICT_REPRESENTATION] = {item: "" for item in last_run.get(LAST_RUN_IDS)}
            last_run[LAST_RUN_IDS].clear()

    return last_run


def email_ec(item):
    """
    Create entry context for an email
    :param item: exchangelib.Item
    :return: entry context dict
    """
    return {
        "CC": None if not item.cc_recipients else [mailbox.email_address for mailbox in item.cc_recipients],
        "BCC": None if not item.bcc_recipients else [mailbox.email_address for mailbox in item.bcc_recipients],
        "To": None if not item.to_recipients else [mailbox.email_address for mailbox in item.to_recipients],
        "From": item.author.email_address,
        "Subject": item.subject,
        "Text": item.text_body,
        "HTML": item.body,
        "HeadersMap": None if not item.headers else {header.name: header.value for header in item.headers},
    }


def is_item_duplicate(item, exclude_ids, incident_filter):
    """
    Checks if an item is a duplicate based on ID and Timestamp.

    RFC 5322 defines Message-ID values as ``<id@domain>``, but in practice the same
    message may appear in different forms across fetches — such as ``id@domain``,
    ``<id@domain>``, ``id@domain>``, or ``<id@domain`` — and they might change between fetches.
    To avoid duplicate incidents, we verify all possible forms in exclude_ids.

    Features:
    1. Smart ID Lookup: Checks both Clean ID (abc) and Bracketed ID (<abc>, <abc, abc>).
    2. Legacy Handling: Handles cases where stored value is "" (if last run is list not dict).
    3. Timestamp Logic: Compares stored time vs item time.

    Returns:
        tuple[bool, str | None]: A tuple containing:
            - is_duplicate (bool): True if item is a duplicate (skip it), False if it should be processed.
            - stored_time (str | None): The stored fetch time for the item, or None if not found/not a duplicate.
    """
    if not item.message_id or not exclude_ids:
        return False, None

    clean_id = item.message_id.strip().strip("<>")

    found_key = None
    for candidate in (clean_id, f"<{clean_id}>", f"{clean_id}>", f"<{clean_id}"):
        if candidate in exclude_ids:
            found_key = candidate
            break

    if found_key is None:
        return False, None
    demisto.debug(f"Dedup match: {item.message_id=}, {found_key=}, {clean_id=}")
    stored_time = exclude_ids[found_key]

    # If stored_time is "" or None, it means it was from an old fetch (List format).
    # We treat it as an existing duplicate and SKIP it.
    if not stored_time:
        return True, stored_time

    current_item_time = (
        item.datetime_created.ewsformat() if incident_filter == RECEIVED_FILTER else item.last_modified_time.ewsformat()
    )
    return stored_time >= current_item_time, stored_time


def parse_item_as_dict(item, email_address=None, camel_case=False, compact_fields=False):  # pragma: no cover
    """
    Parses an exchangelib item as a dict
    :param item: exchangelib.Item to parse
    :param (Optional) email_address: string mailbox
    :param (Optional) camel_case: Is camel case
    :param (Optional) compact_fields: Is compact fields
    :return: Item as a dict
    """

    def parse_object_as_dict(obj):
        raw_dict = {}
        if obj is not None:
            for field in obj.FIELDS:
                raw_dict[field.name] = getattr(obj, field.name, None)
        return raw_dict

    def parse_folder_as_json(folder):  # pragma: no cover
        raw_dict = parse_object_as_dict(folder)
        if "parent_folder_id" in raw_dict:
            raw_dict["parent_folder_id"] = parse_folder_as_json(raw_dict["parent_folder_id"])
        if "effective_rights" in raw_dict:
            raw_dict["effective_rights"] = parse_object_as_dict(raw_dict["effective_rights"])
        return raw_dict

    raw_dict = {}
    for field, value in item._field_vals():
        if type(value) in [str, str, int, float, bool, Body, HTMLBody, None]:
            raw_dict[field] = value
    raw_dict["id"] = item.id
    demisto.debug(f"checking for attachments in email with id {item.id}")
    if getattr(item, "attachments", None):
        raw_dict["attachments"] = [parse_attachment_as_dict(item.id, x) for x in item.attachments]

    for time_field in [
        "datetime_sent",
        "datetime_created",
        "datetime_received",
        "last_modified_time",
        "reminder_due_by",
    ]:
        value = getattr(item, time_field, None)
        if value:
            raw_dict[time_field] = value.ewsformat()

    for dict_field in [
        "effective_rights",
        "parent_folder_id",
        "conversation_id",
        "author",
        "extern_id",
        "received_by",
        "received_representing",
        "reply_to",
        "sender",
        "folder",
    ]:
        value = getattr(item, dict_field, None)
        if value:
            if isinstance(value, list):
                raw_dict[dict_field] = []
                for single_val in value:
                    raw_dict[dict_field].append(parse_object_as_dict(single_val))
            else:
                raw_dict[dict_field] = parse_object_as_dict(value)

    for list_dict_field in ["headers", "cc_recipients", "to_recipients"]:
        value = getattr(item, list_dict_field, None)
        if value:
            raw_dict[list_dict_field] = [parse_object_as_dict(x) for x in value]

    for list_str_field in ["categories"]:
        value = getattr(item, list_str_field, None)
        if value:
            raw_dict[list_str_field] = value

    if getattr(item, "folder", None):
        raw_dict["folder"] = parse_folder_as_json(item.folder)
        folder_path = (
            item.folder.absolute[len(TOIS_PATH) :] if item.folder.absolute.startswith(TOIS_PATH) else item.folder.absolute
        )
        raw_dict["folder_path"] = folder_path

    if compact_fields:
        new_dict = {}
        # noinspection PyListCreation
        fields_list = [
            "datetime_created",
            "datetime_received",
            "datetime_sent",
            "sender",
            "has_attachments",
            "importance",
            "message_id",
            "last_modified_time",
            "size",
            "subject",
            "text_body",
            "headers",
            "body",
            "folder_path",
            "is_read",
            "categories",
        ]

        if "id" in raw_dict:
            new_dict["itemId"] = raw_dict["id"]
            fields_list.append("itemId")

        for field in fields_list:
            if field in raw_dict:
                new_dict[field] = raw_dict.get(field)
        for field in ["received_by", "author", "sender"]:
            if field in raw_dict:
                new_dict[field] = raw_dict.get(field, {}).get("email_address")
        for field in ["to_recipients"]:
            if field in raw_dict:
                new_dict[field] = [x.get("email_address") for x in raw_dict[field]]
        attachments = raw_dict.get("attachments")
        if attachments and len(attachments) > 0:
            file_attachments = [x for x in attachments if x[ATTACHMENT_TYPE] == FILE_ATTACHMENT_TYPE]
            if len(file_attachments) > 0:
                new_dict["FileAttachments"] = file_attachments
            item_attachments = [x for x in attachments if x[ATTACHMENT_TYPE] == ITEM_ATTACHMENT_TYPE]
            if len(item_attachments) > 0:
                new_dict["ItemAttachments"] = item_attachments

        raw_dict = new_dict

    if camel_case:
        raw_dict = keys_to_camel_case(raw_dict)

    if email_address:
        raw_dict[MAILBOX] = email_address
    return raw_dict


def get_entry_for_file_attachment(item_id, attachment):
    """
    Creates a file entry for an attachment
    :param item_id: item_id of the attachment
    :param attachment: attachment dict
    :return: file entry dict for attachment
    """
    entry = fileResult(
        get_attachment_name(attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline),
        attachment.content,
    )
    entry["EntryContext"] = {
        CONTEXT_UPDATE_EWS_ITEM_FOR_ATTACHMENT + CONTEXT_UPDATE_FILE_ATTACHMENT: parse_attachment_as_dict(item_id, attachment)
    }
    return entry


def parse_attachment_as_dict(item_id, attachment):
    """
    Creates a note entry for an attachment
    :param item_id: item_id of the attachment
    :param attachment: attachment dict
    :return: note entry dict for attachment
    """
    try:
        attachment_content = attachment.content if isinstance(attachment, FileAttachment) else attachment.item.mime_content
        if isinstance(attachment_content, str):  # Strings must be encoded before hashing
            attachment_content = attachment_content.encode()
        return {
            ATTACHMENT_ORIGINAL_ITEM_ID: item_id,
            ATTACHMENT_ID: attachment.attachment_id.id,
            "attachmentName": get_attachment_name(
                attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline
            ),
            "attachmentSHA256": hashlib.sha256(attachment_content).hexdigest() if attachment_content else None,
            "attachmentContentType": attachment.content_type,
            "attachmentContentId": attachment.content_id,
            "attachmentContentLocation": attachment.content_location,
            "attachmentSize": attachment.size,
            "attachmentLastModifiedTime": attachment.last_modified_time.ewsformat(),
            "attachmentIsInline": attachment.is_inline,
            ATTACHMENT_TYPE: FILE_ATTACHMENT_TYPE if isinstance(attachment, FileAttachment) else ITEM_ATTACHMENT_TYPE,
        }
    except TypeError as e:
        if str(e) != "must be string or buffer, not None":
            raise
        return {
            ATTACHMENT_ORIGINAL_ITEM_ID: item_id,
            ATTACHMENT_ID: attachment.attachment_id.id,
            "attachmentName": get_attachment_name(
                attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline
            ),
            "attachmentSHA256": None,
            "attachmentContentType": attachment.content_type,
            "attachmentContentId": attachment.content_id,
            "attachmentContentLocation": attachment.content_location,
            "attachmentSize": attachment.size,
            "attachmentLastModifiedTime": attachment.last_modified_time.ewsformat(),
            "attachmentIsInline": attachment.is_inline,
            ATTACHMENT_TYPE: FILE_ATTACHMENT_TYPE if isinstance(attachment, FileAttachment) else ITEM_ATTACHMENT_TYPE,
        }


def get_entry_for_item_attachment(item_id, attachment, target_email):  # pragma: no cover
    """
    Creates a note entry for an item attachment
    :param item_id: Item id
    :param attachment: exchangelib attachment
    :param target_email: target email
    :return: note entry dict for item attachment
    """
    item = attachment.item
    dict_result = parse_attachment_as_dict(item_id, attachment)
    dict_result.update(parse_item_as_dict(item, target_email, camel_case=True, compact_fields=True))
    title = (
        f'EWS get attachment got item for "{target_email}", '
        f'"{get_attachment_name(attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline)}"'  # noqa: E501
    )

    return get_entry_for_object(
        title,
        CONTEXT_UPDATE_EWS_ITEM_FOR_ATTACHMENT + CONTEXT_UPDATE_ITEM_ATTACHMENT,
        dict_result,
    )


""" Command Functions """


def fetch_attachments_for_message(client: EWSClient, args: dict):  # pragma: no cover
    """
    Fetches attachments for a message
    :param client: EWS Client
    :param args: dict of command arguments
    :return: list of parsed entries
    """
    item_id = args.get("item_id", "")
    target_mailbox = args.get("target_mailbox")
    attachment_ids = args.get("attachment_ids")
    identifiers_filter = args.get("identifiers_filter", "")
    identifiers_filter = argToList(identifiers_filter)
    attachment_ids = argToList(attachment_ids)
    account = client.get_account(target_mailbox)
    attachments = client.get_attachments_for_item(item_id, account, attachment_ids)
    entries = []
    for attachment in attachments:
        if isinstance(attachment, FileAttachment):
            try:
                if attachment.content:
                    entries.append(get_entry_for_file_attachment(item_id, attachment))
            except TypeError as e:
                if str(e) != "must be string or buffer, not None":
                    raise
        else:
            entries.append(get_entry_for_item_attachment(item_id, attachment, account.primary_smtp_address))
            if attachment.item.mime_content:
                entries.append(
                    fileResult(
                        get_attachment_name(
                            attachment_name=attachment.name,
                            eml_extension=True,
                            content_id=attachment.content_id,
                            is_inline=attachment.is_inline,
                        ),
                        attachment.item.mime_content,
                    )
                )

    return entries


def search_items_in_mailbox(client: EWSClient, args: dict):  # pragma: no cover
    """
    Search items in mailbox
    :param client: EWS Client
    :param args: dict of command arguments
    :return: Output tuple
    """
    query = args.get("query")
    message_id = args.get("message_id")
    folder_path = args.get("folder_path", "")
    limit = args.get("limit", 100)
    target_mailbox = args.get("target_mailbox")
    is_public = args.get("is_public")
    selected_fields = args.get("selected_fields", "all")
    if not query and not message_id:
        return_error("Missing required argument. Provide query or message-id")

    if message_id and message_id[0] != "<" and message_id[-1] != ">":
        message_id = f"<{message_id}>"

    account = client.get_account(target_mailbox)
    limit = int(limit)
    if folder_path.lower() == "inbox":
        folders = [account.inbox]
    elif folder_path:
        is_public = client.is_default_folder(folder_path, is_public)
        folders = [client.get_folder_by_path(folder_path, account, is_public)]
    else:
        folders = account.inbox.parent.walk()  # pylint: disable=E1101

    items = []  # type: ignore
    selected_all_fields = selected_fields == "all"

    if selected_all_fields:
        restricted_fields = [x.name for x in Message.FIELDS]
    else:
        restricted_fields = set(argToList(selected_fields))  # type: ignore
        restricted_fields.update(["id", "message_id"])  # type: ignore

    for folder in folders:
        if Message not in folder.supported_item_models:
            continue
        if query:
            items_qs = folder.filter(query).only(*restricted_fields)
        else:
            items_qs = folder.filter(message_id=message_id).only(*restricted_fields)
        items += get_limited_number_of_messages_from_qs(items_qs, limit)
        if len(items) >= limit:
            break

    items = items[:limit]
    searched_items_result = [
        parse_item_as_dict(
            item,
            account.primary_smtp_address,
            camel_case=True,
            compact_fields=selected_all_fields,
        )
        for item in items
    ]

    if not selected_all_fields:
        searched_items_result = [
            {k: v for (k, v) in i.items() if k in keys_to_camel_case(restricted_fields)} for i in searched_items_result
        ]

        for item in searched_items_result:
            item["itemId"] = item.pop("id", "")

    hr_items = escape_hr_item_ids([{**item} for item in searched_items_result])
    readable_output = tableToMarkdown(
        "Searched items",
        hr_items,
        headers=ITEMS_RESULTS_HEADERS if selected_all_fields else None,
    )
    output = {CONTEXT_UPDATE_EWS_ITEM: searched_items_result}
    return readable_output, output, searched_items_result


def get_contacts(client: EWSClient, args: dict):  # pragma: no cover
    """
    Retrieve contacts of the target mailbox or client mailbox
    :param client: EWS Client
    :param args: dict of command arguments
    :return:
    """
    limit = args.get("limit", 100)
    target_mailbox = args.get("target_mailbox")

    def parse_physical_address(address):
        result = {}
        for attr in ["city", "country", "label", "state", "street", "zipcode"]:
            result[attr] = getattr(address, attr, None)
        return result

    def parse_phone_number(phone_number):
        result = {}
        for attr in ["label", "phone_number"]:
            result[attr] = getattr(phone_number, attr, None)
        return result

    def parse_contact(contact):
        contact_dict = {
            k: v if not isinstance(v, EWSDateTime) else v.ewsformat()
            for k, v in list(contact._field_vals())
            if isinstance(v, str | EWSDateTime)
        }
        if isinstance(contact, Contact) and contact.physical_addresses:
            contact_dict["physical_addresses"] = list(map(parse_physical_address, contact.physical_addresses))
        if isinstance(contact, Contact) and contact.phone_numbers:
            contact_dict["phone_numbers"] = list(map(parse_phone_number, contact.phone_numbers))
        if isinstance(contact, Contact) and contact.email_addresses and len(contact.email_addresses) > 0:
            contact_dict["emailAddresses"] = [x.email for x in contact.email_addresses]
        contact_dict = keys_to_camel_case(contact_dict)
        contact_dict = {k: v for k, v in list(contact_dict.items()) if v}
        contact_dict.pop("mimeContent", None)
        contact_dict["originMailbox"] = target_mailbox
        return contact_dict

    account = client.get_account(target_mailbox)
    contacts = []

    for contact in account.contacts.all()[: int(limit)]:  # pylint: disable=E1101
        contacts.append(parse_contact(contact))
    readable_output = tableToMarkdown(f"Email contacts for {target_mailbox}", contacts)
    output = {"Account.Email(val.Address == obj.originMailbox).EwsContacts": contacts}
    return readable_output, output, contacts


def find_folders(client: EWSClient, args: dict):
    """
    Finds folders in the mailbox
    :param client: EWS Client
    :param args: dict of command arguments
    :return: Output tuple
    """
    target_mailbox = args.get("target_mailbox")
    account = client.get_account(target_mailbox)
    root = account.root

    if client.is_public_folder:
        root = account.public_folders_root
    folders = []
    for f in root.walk():  # pylint: disable=E1101
        folder = folder_to_context_entry(f)
        folders.append(folder)
    folders_tree = root.tree()  # pylint: disable=E1101
    readable_output = folders_tree
    output = {"EWS.Folders(val.id == obj.id)": folders}
    return readable_output, output, folders


def get_items_from_folder(client: EWSClient, args: dict):  # pragma: no cover
    """
    Retrieve items from folder path
    :param client: EWS Client
    :param args: dict of command arguments
    :return: Output tuple
    """
    folder_path = args.get("folder_path", "")
    limit = args.get("limit", 100)
    target_mailbox = args.get("target_mailbox")
    is_public = args.get("is_public")
    get_internal_item = args.get("get_internal_item", "no")
    account = client.get_account(target_mailbox)
    limit = int(limit)
    get_internal_item = get_internal_item == "yes"
    is_public = client.is_default_folder(folder_path, is_public)
    folder = client.get_folder_by_path(folder_path, account, is_public)
    qs = folder.filter().order_by("-datetime_created")[:limit]
    items = get_limited_number_of_messages_from_qs(qs, limit)
    items_result = []

    for item in items:
        item_attachment = parse_item_as_dict(item, account.primary_smtp_address, camel_case=True, compact_fields=True)
        for attachment in item.attachments:
            if get_internal_item and isinstance(attachment, ItemAttachment) and isinstance(attachment.item, Message):
                # if found item attachment - switch item to the attchment
                item_attachment = parse_item_as_dict(
                    attachment.item,
                    account.primary_smtp_address,
                    camel_case=True,
                    compact_fields=True,
                )
                break
        items_result.append(item_attachment)

    hm_headers = [
        "sender",
        "subject",
        "hasAttachments",
        "datetimeReceived",
        "receivedBy",
        "author",
        "toRecipients",
        "itemId",
    ]
    hr_items = escape_hr_item_ids([{**item} for item in items_result])
    readable_output = tableToMarkdown("Items in folder " + folder_path, hr_items, headers=hm_headers)
    output = {CONTEXT_UPDATE_EWS_ITEM: items_result}
    return readable_output, output, items_result


def get_items(client: EWSClient, args: dict):  # pragma: no cover
    """
    Get items from target mailbox or client mailbox
    :param client: EWS Client
    :param args: dict of command arguments
    :return:
    """
    item_ids = args.get("item_ids", "")
    target_mailbox = args.get("target_mailbox")
    item_ids = argToList(item_ids)
    account = client.get_account(target_mailbox)
    items = client.get_items_from_mailbox(account, item_ids)
    items = [x for x in items if isinstance(x, Message)]
    items_as_incidents = [parse_incident_from_item(x) for x in items]
    items_to_context = [parse_item_as_dict(x, account.primary_smtp_address, True, True) for x in items]
    hr_items = escape_hr_item_ids([{**item} for item in items_to_context])
    readable_output = tableToMarkdown("Get items", hr_items, ITEMS_RESULTS_HEADERS)
    output = {
        CONTEXT_UPDATE_EWS_ITEM: items_to_context,
        "Email": [email_ec(item) for item in items],
    }
    return readable_output, output, items_as_incidents


def collect_manual_attachments(manualAttachObj):  # pragma: no cover
    """Collect all manual attachments' data

    Args:
        manualAttachObj (str): String representation of the manually attached files list.

    Returns:
        List[Dict]. List of the files data.
    """
    manually_attached_objects = argToList(manualAttachObj)

    attachments = []
    for attachment in manually_attached_objects:
        file_res = demisto.getFilePath(os.path.basename(attachment["RealFileName"]))

        path = file_res["path"]

        with open(path, "rb") as fp:
            data = fp.read()

        attachments.append({"name": attachment["FileName"], "data": data, "cid": ""})

    return attachments


def collect_attachments(attachments_ids, attachments_cids, attachments_names):  # pragma: no cover
    """Collect all attachments' data

    Args:
        attachments_ids (str): String representation of the files ids list.
        attachments_cids (str): String representation of the files content ids list.
        attachments_names (str): String representation of the files names list.

    Returns:
        List[Dict]. List of the files data.
    """
    attachments = []

    files_ids = argToList(attachments_ids)
    files_cids = argToList(attachments_cids)
    files_names = argToList(attachments_names)

    for index, file_id in enumerate(files_ids):
        try:
            file_res = demisto.getFilePath(file_id)
            path = file_res["path"]

            filename = files_names[index] if len(files_names) > index and files_names[index] else file_res["name"]

            cid = files_cids[index] if len(files_cids) > index and files_cids[index] else ""

            with open(path, "rb") as fp:
                data = fp.read()

            attachments.append({"name": filename, "data": data, "cid": cid})

        except Exception as e:
            demisto.error(f"Invalid entry {file_id} with exception: {e}")
            return_error(f"Entry {file_id} is not valid or is not a file entry")

    return attachments


def handle_transient_files(transient_files, transient_files_contents, transient_files_cids):
    """Creates the transient attachments data

    Args:
        transient_files (str): String representation of the transient files names list.
        transient_files_contents (str): String representation of the transient files content list.
        transient_files_cids (str): String representation of the transient files content ids list.

    Returns:
        List[Dict]. List of the transient files data.
    """
    transient_attachments = []

    files_names = argToList(transient_files)
    files_contents = argToList(transient_files_contents)
    files_cids = argToList(transient_files_cids)

    for index in range(len(files_names)):
        file_name = files_names[index]

        if index >= len(files_contents):
            break

        file_content = bytes(files_contents[index], UTF_8)

        file_cid = "" if index >= len(files_cids) else files_cids[index]

        transient_attachments.append({"name": file_name, "data": file_content, "cid": file_cid})

    return transient_attachments


def handle_template_params(template_params):  # pragma: no cover
    """Translates the template params if they exist from the context

    Args:
        template_params (str): JSON string that represent the variables names to be replaced and the desired value.
                                Value can be either real value or context key to fetch the value from.

    Returns:
        Dict. `variable_name: value_to_use` of the templated parameters.
    """
    actual_params = {}

    if template_params:
        try:
            params = json.loads(template_params)

            for p in params:
                if params[p].get("value"):
                    actual_params[p] = params[p]["value"]
                elif params[p].get("key"):
                    actual_params[p] = demisto.dt(demisto.context(), params[p]["key"])
        except ValueError as e:
            return_error(f"Unable to parse template_params: {e!s}")

    return actual_params


def create_message_object(to, cc, bcc, subject, body, additional_headers, from_address, reply_to, importance):
    """Creates the message object according to the existence of additional custom headers."""
    if additional_headers:
        return Message(
            to_recipients=to,
            author=from_address,
            cc_recipients=cc,
            bcc_recipients=bcc,
            subject=subject,
            reply_to=reply_to,
            body=body,
            importance=importance,
            **additional_headers,
        )

    return Message(
        to_recipients=to,
        author=from_address,
        cc_recipients=cc,
        bcc_recipients=bcc,
        subject=subject,
        reply_to=reply_to,
        body=body,
        importance=importance,
    )


def create_message(
    to,
    handle_inline_image: bool = True,
    subject="",
    body="",
    bcc=None,
    cc=None,
    html_body=None,
    attachments=[],
    additional_headers=None,
    from_address=None,
    reply_to=None,
    importance=None,
):  # pragma: no cover
    """Creates the Message object that will be sent.

    Args:
        to (list): Main recipients.
        cc (list): CC recipients.
        bcc (list): BCC recipients.
        subject (str): Email's subject.
        body (str): Email's simple text body.
        html_body (str): Email's html body.
        attachments (list): Files to be attached to the mail, both inline and as files.
        additional_headers (Dict): Custom headers to be added to the message.
        from_address (str): The email address from which to reply.
        reply_to (list): Email addresses that need to be used to reply to the message.
        handle_inline_image (bool): Whether to handle inline images in the HTML body.
    Returns:
        Message. Message object ready to be sent.
    """
    demisto.debug(f"create_message: Received {len(attachments)} attachments, {handle_inline_image=}")
    if not html_body:
        # This is a simple text message - we cannot have CIDs here
        message = create_message_object(to, cc, bcc, subject, body, additional_headers, from_address, reply_to, importance)

        for attachment in attachments:
            if not attachment.get("cid"):
                new_attachment = FileAttachment(name=attachment.get("name"), content=attachment.get("data"))
                message.attach(new_attachment)

    else:
        html_attachments: list = []
        if handle_inline_image:
            html_body, html_attachments = handle_html(html_body)
            attachments += html_attachments
            demisto.debug(f"create_message: Processed HTML body with {len(attachments)} attachments")
        message = create_message_object(
            to, cc, bcc, subject, HTMLBody(html_body), additional_headers, from_address, reply_to, importance
        )

        for attachment in attachments:
            if not isinstance(attachment, FileAttachment):
                if not attachment.get("cid"):
                    attachment = FileAttachment(name=attachment.get("name"), content=attachment.get("data"))
                else:
                    attachment = FileAttachment(
                        name=attachment.get("name"),
                        content=attachment.get("data"),
                        is_inline=True,
                        content_id=attachment.get("cid"),
                    )

            message.attach(attachment)

    return message


def add_additional_headers(additional_headers):
    """Adds custom headers to the Message object

    Args:
        additional_headers (str): Headers list as string. Example: headerName1=headerValue1,headerName2=headerValue2

    Returns:
        Dict. Headers dictionary in the form of: `header_name: header value`
    """
    headers = {}

    for header in argToList(additional_headers):
        header_name, header_value = header.split("=", 1)

        class TempClass(ExtendedProperty):
            distinguished_property_set_id = "InternetHeaders"
            property_name = header_name
            property_type = "String"

        try:
            Message.register(header_name, TempClass)
            headers[header_name] = header_value
        except ValueError:
            Message.deregister(header_name)
            try:
                Message.register(header_name, TempClass)
                headers[header_name] = header_value
            except ValueError as e:
                demisto.debug("EWSO365 - Header " + header_name + " could not be registered. " + str(e))

    return headers


def send_email(client: EWSClient, args: dict):  # pragma: no cover
    to = argToList(args.get("to"))
    subject = args.get("subject", "")
    body = args.get("body", "")
    bcc = argToList(args.get("bcc"))
    cc = argToList(args.get("cc"))
    htmlBody = args.get("htmlBody")
    attachIDs = args.get("attachIDs", "")
    attachCIDs = args.get("attachCIDs", "")
    attachNames = args.get("attachNames", "")
    manualAttachObj = args.get("manualAttachObj")
    transientFile = args.get("transientFile")
    transientFileContent = args.get("transientFileContent")
    transientFileCID = args.get("transientFileCID")
    templateParams = args.get("templateParams")
    additionalHeader = args.get("additionalHeader")
    raw_message = args.get("raw_message")
    from_address = args.get("from_address")
    reply_to = argToList(args.get("replyTo"))
    importance = args.get("importance")
    render_body = argToBoolean(args.get("renderBody", False))
    handle_inline_image = argToBoolean(args.get("handle_inline_image", True))

    # Basic validation - we allow pretty much everything but you have to have at least a recipient
    # We allow messages without subject and also without body
    if not to and not cc and not bcc:
        return_error("You must have at least one recipient")

    if raw_message:
        message = Message(
            to_recipients=to,
            cc_recipients=cc,
            bcc_recipients=bcc,
            body=raw_message,
            author=from_address,
            reply_to=reply_to,
            importance=importance,
        )

    else:
        if additionalHeader:
            additionalHeader = add_additional_headers(additionalHeader)

        # collect all types of attachments
        attachments = collect_attachments(attachIDs, attachCIDs, attachNames)
        attachments.extend(collect_manual_attachments(manualAttachObj))
        attachments.extend(handle_transient_files(transientFile, transientFileContent, transientFileCID))

        # update body and html_body with the templated params, if exists
        template_params = handle_template_params(templateParams)
        if template_params:
            body = body.format(**template_params)
            if htmlBody:
                htmlBody = htmlBody.format(**template_params)

        message = create_message(
            to,
            handle_inline_image,
            subject,
            body,
            bcc,
            cc,
            htmlBody,
            attachments,
            additionalHeader,
            from_address,
            reply_to,
            importance,
        )

    client.send_email(message)

    results = [CommandResults(entry_type=EntryType.NOTE, raw_response="Mail sent successfully")]
    if render_body:
        results.append(
            CommandResults(
                entry_type=EntryType.NOTE,
                content_format=EntryFormat.HTML,
                raw_response=htmlBody,
            )
        )

    return results


def reply_mail(client: EWSClient, args: dict):  # pragma: no cover
    to = argToList(args.get("to", ""))
    inReplyTo = args.get("inReplyTo", "")
    subject = args.get("subject", "")
    body = args.get("body", "")
    bcc = argToList(args.get("bcc"))
    cc = argToList(args.get("cc"))
    htmlBody = args.get("htmlBody")
    attachIDs = args.get("attachIDs", "")
    attachCIDs = args.get("attachCIDs", "")
    attachNames = args.get("attachNames", "")
    manualAttachObj = args.get("manualAttachObj")
    handle_inline_image: bool = argToBoolean(args.get("handle_inline_image", True))
    # collect all types of attachments
    attachments = collect_attachments(attachIDs, attachCIDs, attachNames)
    attachments.extend(collect_manual_attachments(manualAttachObj))
    client.reply_email(
        in_reply_to=inReplyTo,
        to=to,
        body=body,
        subject=subject,
        bcc=bcc,
        cc=cc,
        html_body=htmlBody,
        attachments=attachments,
        handle_inline_image=handle_inline_image,
    )


def get_item_as_eml(client: EWSClient, args: dict):  # pragma: no cover
    """
    Retrieve item as an eml
    :param client: EWS Client
    :param args: dict of command arguments
    :return: Output tuple
    """
    item_id = args.get("item_id", "")
    target_mailbox = args.get("target_mailbox")
    account = client.get_account(target_mailbox)
    item = client.get_item_from_mailbox(account, item_id)

    if item.mime_content:
        email_content = cast_mime_item_to_message(item)
        if item.headers:
            # compare header keys case-insensitive
            attached_email_headers = [
                (h.lower(), " ".join(map(str.strip, v.split("\r\n")))) for (h, v) in list(email_content.items())
            ]
            for header in item.headers:
                if (
                    header.name.lower(),
                    header.value,
                ) not in attached_email_headers and header.name.lower() != "content-type":
                    try:
                        email_content.add_header(header.name, header.value)
                    except ValueError as err:
                        if "There may be at most" not in str(err):
                            raise err

        eml_name = item.subject if item.subject else "demisto_untitled_eml"
        email_data = decode_email_data(email_content)
        file_result = fileResult(eml_name + ".eml", email_data)
        file_result = file_result if file_result else "Failed uploading eml file to war room"

        return file_result
    return None


def handle_attached_email_with_incorrect_message_id(attached_email: Message):
    """This function handles a malformed Message-ID value which can be returned in the header of certain email objects.
    This issue happens due to a current bug in "email" library and further explained in XSUP-32074.
    Public issue link: https://github.com/python/cpython/issues/105802

    The function will run on every attached email if exists, check its Message-ID header value and fix it if possible.
    Args:
        attached_email (Message): attached email object.

    Returns:
        Message: attached email object.
    """
    message_id_value = ""
    for i in range(len(attached_email._headers)):
        if attached_email._headers[i][0].lower() == "message-id":
            message_id = attached_email._headers[i][1]
            message_header = attached_email._headers[i][0]
            demisto.debug(f"Handling Message-ID header, {message_id=}.")
            try:
                message_id_value = handle_incorrect_message_id(message_id)
                if message_id_value != message_id:
                    # If the Message-ID header was fixed in the context of this function
                    # the header will be replaced in _headers list
                    attached_email._headers.pop(i)
                    attached_email._headers.append((message_header, message_id_value))

            except Exception as e:
                # The function is designed to handle a specific format error for the Message-ID header
                # as explained in the docstring.
                # That being said, we do expect the header to be in a known format.
                # If this function encounters a header format which is not in the known format and can't be fixed,
                # the header will be ignored completely to prevent crashing the fetch command.
                demisto.debug(f"Invalid {message_id=}, Error: {e}")
                break
            break
    return attached_email


def handle_attached_email_with_incorrect_from_header(attached_email: Message):
    """This function handles a malformed From value which can be returned in the header of certain email objects.
    This issue happens due to a current bug in "email" library.
    Public issue link: https://github.com/python/cpython/issues/114906

    The function will run on every attached email if exists, check its From header value and fix it if possible.
    Args:
        attached_email (Message): attached email object.

    Returns:
        Message: attached email object.
    """
    for i, (header_name, header_value) in enumerate(attached_email._headers):
        if header_name.lower() == "from":
            demisto.debug(f"Handling From header, value={header_value}.")
            try:
                new_value = parser.get_address_list(header_value)[0].value
                new_value = new_value.replace("\n", " ").replace("\r", " ").strip()
                if header_value != new_value:
                    # Update the 'From' header with the corrected value
                    attached_email._headers[i] = (header_name, new_value)
                    demisto.debug(f"From header fixed, new value: {new_value}")

            except Exception as e:
                demisto.debug(f"Error processing From header: {e}")
            break
    return attached_email


def handle_incorrect_message_id(message_id: str) -> str:
    """
    Use regex to identify and correct one of the following invalid message_id formats:
    1. '<[message_id]>' --> '<message_id>'
    2. '\r\n\t<[message_id]>' --> '\r\n\t<message_id>'
    If no necessary changes identified the original 'message_id' argument value is returned.
    """
    if re.search(r"\<\[.*\]\>", message_id):
        # find and replace "<[" with "<" and "]>" with ">"
        fixed_message_id = re.sub(r"<\[(.*?)\]>", r"<\1>", message_id)
        demisto.debug("Fixed message id {message_id} to {fixed_message_id}")
        return fixed_message_id
    return message_id


def decode_email_data(email_obj: Message):
    attached_email_bytes = email_obj.as_bytes()
    chardet_detection = chardet.detect(attached_email_bytes)
    encoding = chardet_detection.get("encoding", "utf-8") or "utf-8"
    try:
        # Trying to decode using the detected encoding
        data = attached_email_bytes.decode(encoding)
    except UnicodeDecodeError:
        # In case the detected encoding fails apply the default encoding
        demisto.info(
            f'Could not decode attached email using detected encoding: {encoding}, retrying '
            f'using utf-8.\nAttached email details: '
            f'\nMessage-ID = {email_obj.get("Message-ID")}'
            f'\nDate = {email_obj.get("Date")}'
            f'\nSubject = {email_obj.get("Subject")}'
            f'\nFrom = {email_obj.get("From")}'
            f'\nTo = {email_obj.get("To")}'
        )
        try:
            data = attached_email_bytes.decode("utf-8")
        except UnicodeDecodeError:
            demisto.info("Could not decode attached email using utf-8. returned the content without decoding")
            data = attached_email_bytes  # type: ignore

    return data


def cast_mime_item_to_message(item):
    mime_content = item.mime_content
    email_policy = SMTP if mime_content.isascii() else SMTPUTF8

    if isinstance(mime_content, str) and not mime_content.isascii():
        mime_content = mime_content.encode()

    if isinstance(mime_content, bytes):
        message = email.message_from_bytes(mime_content, policy=email_policy)  # type: ignore[arg-type]
    else:
        message = email.message_from_string(mime_content, policy=email_policy)  # type: ignore[arg-type]

    return message


def parse_incident_from_item(item):  # pragma: no cover
    """
    Parses an incident from an item
    :param item: item to parse
    :return: Parsed item
    """
    incident = {}
    labels = []
    demisto.debug(f"starting to parse the email with id {item.id} into an incident")
    try:
        incident["details"] = item.text_body or item.body
    except AttributeError:
        incident["details"] = item.body
    incident["name"] = item.subject
    labels.append({"type": "Email/subject", "value": item.subject})
    incident["occurred"] = item.datetime_received.ewsformat()

    # handle recipients
    if item.to_recipients:
        for recipient in item.to_recipients:
            labels.append({"type": "Email", "value": recipient.email_address})

    # handle cc
    if item.cc_recipients:
        for recipient in item.cc_recipients:
            labels.append({"type": "Email/cc", "value": recipient.email_address})
    # handle email from
    if item.sender:
        labels.append({"type": "Email/from", "value": item.sender.email_address})
        labels.append({"type": "Email/from/name", "value": item.sender.name})

    # email format
    email_format = ""
    try:
        if item.text_body:
            labels.append({"type": "Email/text", "value": item.text_body})
            email_format = "text"
    except AttributeError:
        pass
    if item.body:
        labels.append({"type": "Email/html", "value": item.body})
        email_format = "HTML"
    labels.append({"type": "Email/format", "value": email_format})

    # handle attachments
    if item.attachments:
        incident["attachment"] = []
        demisto.debug(f"parsing {len(item.attachments)} attachments for item with id {item.id}")
        attachment_counter = 0
        for attachment in item.attachments:
            attachment_counter += 1
            demisto.debug(f"retrieving attachment number {attachment_counter} of email with id {item.id}")
            file_result = None
            label_attachment_type = None
            label_attachment_id_type = None
            if isinstance(attachment, FileAttachment):
                try:
                    if attachment.content:
                        # file attachment
                        label_attachment_type = "attachments"
                        label_attachment_id_type = "attachmentId"

                        # save the attachment
                        file_name = get_attachment_name(
                            attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline
                        )
                        demisto.debug(
                            f"saving content number {attachment_counter}, "
                            f"of size {sys.getsizeof(attachment.content)}, of email with id {item.id}"
                        )
                        file_result = fileResult(file_name, attachment.content)

                        # check for error
                        if file_result["Type"] == entryTypes["error"]:
                            demisto.error(file_result["Contents"])
                            raise Exception(file_result["Contents"])

                        # save attachment to incident
                        incident["attachment"].append(
                            {
                                "path": file_result["FileID"],
                                "name": get_attachment_name(
                                    attachment_name=attachment.name,
                                    content_id=attachment.content_id,
                                    is_inline=attachment.is_inline,
                                ),
                                "description": FileAttachmentType.ATTACHED if not attachment.is_inline else "",
                            }
                        )
                except TypeError as e:
                    if str(e) != "must be string or buffer, not None":
                        raise
                    continue
                except SAXParseException as e:
                    # TODO: When a fix is released, we will need to bump the library version.
                    #  https://github.com/ecederstrand/exchangelib/issues/1200
                    demisto.debug(
                        f"An XML error occurred while loading an attachments content."
                        f"\nMessage ID is {item.id}"
                        f"\nError: {e.getMessage()}"
                    )
                    continue
            else:
                # other item attachment
                label_attachment_type = "attachmentItems"
                label_attachment_id_type = "attachmentItemsId"

                # save the attachment
                if attachment.item.mime_content:
                    attached_email = cast_mime_item_to_message(attachment.item)
                    if attachment.item.headers:
                        # compare header keys case-insensitive
                        attached_email_headers = []
                        attached_email = handle_attached_email_with_incorrect_message_id(attached_email)
                        attached_email = handle_attached_email_with_incorrect_from_header(attached_email)
                        for h, v in attached_email.items():
                            if not isinstance(v, str):
                                try:
                                    v = str(v)
                                except:  # noqa: E722
                                    demisto.debug(f'cannot parse the header "{h}"')
                                    continue

                            v = " ".join(map(str.strip, v.split("\r\n")))
                            attached_email_headers.append((h.lower(), v))
                        demisto.debug(f"{attached_email_headers=}")
                        for header in attachment.item.headers:
                            if (
                                header.name.lower(),
                                header.value,
                            ) not in attached_email_headers and header.name.lower() != "content-type":
                                try:
                                    if header.name.lower() == "message-id":
                                        """ Handle a case where a Message-ID header was NOT already in attached_email,
                                        and instead is coming from attachment.item.headers.
                                        Meaning it wasn't handled in handle_attached_email_with_incorrect_message_id function
                                        and instead it is handled here using handle_incorrect_message_id function."""
                                        correct_message_id = handle_incorrect_message_id(header.value)
                                        if (header.name.lower(), correct_message_id) not in attached_email_headers:
                                            attached_email.add_header(header.name, correct_message_id)
                                    else:
                                        attached_email.add_header(header.name, header.value)
                                except ValueError as err:
                                    if "There may be at most" not in str(err):
                                        raise err

                    data = decode_email_data(attached_email)
                    file_result = fileResult(
                        get_attachment_name(
                            attachment_name=attachment.name,
                            eml_extension=True,
                            content_id=attachment.content_id,
                            is_inline=attachment.is_inline,
                        ),
                        data,
                    )

                if file_result:
                    # check for error
                    if file_result["Type"] == entryTypes["error"]:
                        demisto.error(file_result["Contents"])
                        raise Exception(file_result["Contents"])

                    # save attachment to incident
                    incident["attachment"].append(
                        {
                            "path": file_result["FileID"],
                            "name": get_attachment_name(
                                attachment_name=attachment.name,
                                eml_extension=True,
                                content_id=attachment.content_id,
                                is_inline=attachment.is_inline,
                            ),
                        }
                    )

            labels.append(
                {
                    "type": label_attachment_type,
                    "value": get_attachment_name(
                        attachment_name=attachment.name, content_id=attachment.content_id, is_inline=attachment.is_inline
                    ),
                }
            )
            labels.append({"type": label_attachment_id_type, "value": attachment.attachment_id.id})
        demisto.debug(f"finished parsing attachment {attachment_counter} of email with id {item.id}")

    # handle headers
    if item.headers:
        headers = []
        for header in item.headers:
            labels.append(
                {
                    "type": f"Email/Header/{header.name}",
                    "value": str(header.value),
                }
            )
            headers.append(f"{header.name}: {header.value}")
        labels.append({"type": "Email/headers", "value": "\r\n".join(headers)})

    # handle item id
    if item.message_id:
        labels.append({"type": "Email/MessageId", "value": str(item.message_id)})

    if item.id:
        labels.append({"type": "Email/ID", "value": item.id})
        labels.append({"type": "Email/itemId", "value": item.id})

    # handle conversion id
    if item.conversation_id:
        labels.append({"type": "Email/ConversionID", "value": item.conversation_id.id})

    incident["labels"] = labels
    demisto.debug(f"Starting to generate rawJSON for incident, from email with id {item.id}")
    incident["rawJSON"] = json.dumps(parse_item_as_dict(item, None), ensure_ascii=False)
    demisto.debug(f"Finished generating rawJSON from email with id {item.id}")

    return incident


def fetch_emails_as_incidents(client: EWSClient, last_run, incident_filter, skip_unparsable_emails: bool = False):
    """
    Fetch incidents
    :param client: EWS Client
    :param last_run: last run dict
    :return:
    """
    last_run = get_last_run(client, last_run)
    demisto.debug(f"get_last_run: {last_run=}")
    last_fetch_time = last_run.get(LAST_RUN_TIME)
    excluded_ids = last_run.get(LAST_RUN_IDS_DICT_REPRESENTATION)
    try:
        last_emails = fetch_last_emails(
            client,
            client.folder_name,
            last_run.get(LAST_RUN_TIME),
            excluded_ids,
            incident_filter,
        )

        incidents = []
        incident: dict[str, str] = {}
        emails_ids = []  # Used for mark emails as read
        demisto.debug(f"{APP_NAME} - Started fetch with {len(last_emails)} at {last_fetch_time}")
        current_fetch_ids = {}
        last_modification_time = last_fetch_time

        if isinstance(last_modification_time, EWSDateTime):
            last_modification_time = last_modification_time.ewsformat()

        for item in last_emails:
            try:
                if item.message_id:
                    current_fetch_ids[item.message_id] = (
                        item.datetime_created.ewsformat()
                        if incident_filter == RECEIVED_FILTER
                        else item.last_modified_time.ewsformat()
                    )
                    incident = parse_incident_from_item(item)
                    incidents.append(incident)

                    if incident_filter == MODIFIED_FILTER:
                        item_modified_time = item.last_modified_time.ewsformat()
                        if last_modification_time is None or last_modification_time < item_modified_time:
                            last_modification_time = item_modified_time

                    if item.id and item.is_read is False:
                        emails_ids.append(item.id)

                    if len(incidents) >= client.max_fetch:
                        break
                else:
                    demisto.debug(f"Skipped item: item with no message_id {item=}")
            except Exception as e:
                if not skip_unparsable_emails:  # default is to raise and exception and fail the command
                    raise

                # when the skip param is `True`, we log the exceptions and move on instead of failing the whole fetch
                error_msg = (
                    "Encountered email parsing issue while fetching. "
                    f"Skipping item with message id: {item.message_id or '<error parsing message_id>'}"
                )
                demisto.debug(f"{error_msg}, Error: {e!s} {traceback.format_exc()}")
                demisto.updateModuleHealth(error_msg, is_error=False)

        demisto.debug(f"{APP_NAME} - ending fetch - got {len(incidents)} incidents.")

        if incident_filter == MODIFIED_FILTER:
            last_incident_run_time = last_modification_time
        else:  # default case - using 'received' time
            last_incident_run_time = incident.get("occurred", last_fetch_time)

        # making sure both last fetch time and the time of most recent incident are the same type for comparing.
        if isinstance(last_incident_run_time, EWSDateTime):
            last_incident_run_time = last_incident_run_time.ewsformat()

        if isinstance(last_fetch_time, EWSDateTime):
            last_fetch_time = last_fetch_time.ewsformat()

        demisto.debug(
            f"#### last_incident_time: {last_incident_run_time}({type(last_incident_run_time)})."
            f"last_fetch_time: {last_fetch_time}({type(last_fetch_time)}) ####"
        )

        # If the fetch query is not fully fetched (we didn't have any time progress) - then we keep the
        # id's from current fetch until progress is made. This is for when max_fetch < incidents_from_query.
        if not last_incident_run_time or not last_fetch_time or last_incident_run_time > last_fetch_time:
            ids = current_fetch_ids
        else:
            ids = excluded_ids | current_fetch_ids

        new_last_run = {
            LAST_RUN_TIME: last_incident_run_time,
            LAST_RUN_FOLDER: client.folder_name,
            LAST_RUN_IDS_DICT_REPRESENTATION: ids,
            ERROR_COUNTER: 0,
        }

        demisto.debug(f"Set last run to: {new_last_run=}")
        demisto.setLastRun(new_last_run)

        if client.mark_as_read:
            mark_item_as_read(client, {"item_ids": emails_ids})

        return incidents

    except RateLimitError:
        if LAST_RUN_TIME in last_run:
            last_run[LAST_RUN_TIME] = last_run[LAST_RUN_TIME].ewsformat()
        if ERROR_COUNTER not in last_run:
            last_run[ERROR_COUNTER] = 0
        last_run[ERROR_COUNTER] += 1
        demisto.setLastRun(last_run)
        if last_run[ERROR_COUNTER] > 2:
            raise
        return []


def fetch_last_emails(
    client: EWSClient, folder_name="Inbox", since_datetime=None, exclude_ids=None, incident_filter=RECEIVED_FILTER
):
    """
    Fetches last emails
    :param client: EWS client
    :param (Optional) folder_name: folder name to pull from
    :param (Optional) since_datetime: items will be searched after this datetime
    :param (Optional) exclude_ids: exclude ids from fetch
    :return: list of exchangelib.Items
    """
    qs = client.get_folder_by_path(folder_name, is_public=client.is_public_folder)
    demisto.debug(f"Finished getting the folder named {folder_name} by path")
    if since_datetime:
        if incident_filter == RECEIVED_FILTER:
            qs = qs.filter(datetime_received__gte=since_datetime)
        else:
            qs = qs.filter(last_modified_time__gte=since_datetime)
    else:
        tz = EWSTimeZone("UTC")
        first_fetch_datetime = dateparser.parse(FETCH_TIME)
        assert first_fetch_datetime is not None
        first_fetch_ews_datetime = EWSDateTime.from_datetime(first_fetch_datetime.replace(tzinfo=tz))
        demisto.debug(f"{first_fetch_ews_datetime=}")
        if incident_filter == RECEIVED_FILTER:
            qs = qs.filter(datetime_received__gte=first_fetch_ews_datetime)
        else:
            qs = qs.filter(last_modified_time__gte=first_fetch_ews_datetime)

    qs = qs.filter().only(*[x.name for x in Message.FIELDS if x.name.lower() != "mime_content"])
    if incident_filter == RECEIVED_FILTER:
        qs = qs.filter().order_by("datetime_received")
    else:
        qs = qs.filter().order_by("last_modified_time")

    result = []
    exclude_ids = exclude_ids if exclude_ids else {}
    demisto.debug(f"{APP_NAME} - Exclude ID list: {exclude_ids}")
    qs.chunk_size = min(client.max_fetch, 100)
    qs.page_size = min(client.max_fetch, 100)
    demisto.debug("Before iterating on queryset")
    demisto.debug(f"Size of the queryset object in fetch-incidents: {sys.getsizeof(qs)}")
    for item in qs:
        demisto.debug("next iteration of the queryset in fetch-incidents")
        if isinstance(item, Message):
            is_duplicate, previous_fetch_time = is_item_duplicate(item, exclude_ids, incident_filter)
            if is_duplicate:
                received_time = item.datetime_created.ewsformat()
                modified_time = item.last_modified_time.ewsformat()
                demisto.debug(
                    f"{item.subject=} with {item.message_id=} was excluded. previous fetch time: "
                    f"{previous_fetch_time}, (if no time - because of the transition from list to dict). "
                    f"current fetch time: {received_time if incident_filter == RECEIVED_FILTER else modified_time}"
                )
                continue
            demisto.debug(f"Appending {item.subject=} with {item.message_id=}")
            result.append(item)
            if len(result) >= client.max_fetch:
                break
    demisto.debug(f"{APP_NAME} - Got total of {len(result)} from ews query.")
    return result


def test_module(client: EWSClient, max_fetch):  # pragma: no cover
    """
    test-module
    * Max incidents per fetch <= MAX_INCIDENTS_PER_FETCH
    * Account can be retrieved
    * Account has read rights
    * Test access to fetch folder
    :param client: EWS Client
    :param max_fetch: Max fetches per incident
    :return: "ok"
    """
    try:
        if int(max_fetch) > MAX_INCIDENTS_PER_FETCH:
            return_error(
                f"Error - Max incidents per fetch cannot be greater than {MAX_INCIDENTS_PER_FETCH}. You provided: {max_fetch}"
            )
        account = client.get_account()
        if not account.root.effective_rights.read:  # pylint: disable=E1101
            raise Exception(
                "Success to authenticate, but user has no permissions to read from the mailbox. "
                "Need to delegate the user permissions to the mailbox - "
                "please read integration documentation and follow the instructions"
            )
        client.get_folder_by_path(client.folder_name, account, client.is_public_folder).test_access()
    except ErrorFolderNotFound as e:
        if "Top of Information Store" in str(e):
            raise Exception(
                "Success to authenticate, but user probably has no permissions to read from the specific folder."
                "Check user permissions. You can try !ews-find-folders command to "
                "get all the folders structure that the user has permissions to"
            )
    except Exception as e:
        if "403" in str(e):
            error_message_simple = (
                "Got invalid response with status code: 403."
                " Please make sure you have the right permissions to your application.\n"
            )
            raise DemistoException(error_message_simple)
        raise DemistoException(str(e))
    return "ok"


def sub_main():  # pragma: no cover
    is_test_module = False
    params = demisto.params()
    args = prepare_args(demisto.args())
    # client's default_target_mailbox is the authorization source for the instance
    params["default_target_mailbox"] = args.get(
        "target_mailbox", args.get("source_mailbox", params.get("default_target_mailbox", ""))
    )
    if params.get("upn_mailbox") and not (args.get("target_mailbox")):
        params["default_target_mailbox"] = params.get("upn_mailbox", "")

    if params.get("access_type") == "Impersonation":
        demisto.info(
            "Note: The access type Impersonation you are using is deprecated. For more information, "
            "please refer to the integration description."
        )
    try:
        client = get_client_from_params(params)
        start_logging()
        # replace sensitive access_token value in logs
        if not isinstance(client.credentials, CustomDomainOAuth2Credentials):  # Should not fail
            raise DemistoException("Failed to initialize EWS Client properly, check credentials")

        add_sensitive_log_strs(client.credentials.access_token.get("access_token", ""))
        command = demisto.command()
        # commands that return a single note result
        normal_commands = {
            "ews-get-searchable-mailboxes": get_searchable_mailboxes,
            "ews-move-item": move_item,
            "ews-delete-items": delete_items,
            "ews-search-mailbox": search_items_in_mailbox,
            "ews-get-contacts": get_contacts,
            "ews-get-out-of-office": get_out_of_office_state,
            "ews-recover-messages": recover_soft_delete_item,
            "ews-create-folder": create_folder,
            "ews-mark-item-as-junk": mark_item_as_junk,
            "ews-find-folders": find_folders,
            "ews-get-items-from-folder": get_items_from_folder,
            "ews-get-items": get_items,
            "ews-get-folder": get_folder,
            "ews-expand-group": get_expanded_group,
            "ews-mark-items-as-read": mark_item_as_read,
            "ews-delete-attachment": delete_attachments_for_message,
            "ews-get-attachment": fetch_attachments_for_message,
        }

        # commands that may return multiple results or non-note result
        special_output_commands = {
            "ews-get-items-as-eml": get_item_as_eml,
            "reply-mail": reply_mail,
        }
        # system commands:
        if command == "test-module":
            is_test_module = True
            demisto.results(test_module(client, params.get("max_fetch")))
        elif command == "fetch-incidents":
            last_run = demisto.getLastRun()
            incident_filter = params.get("incidentFilter", RECEIVED_FILTER)
            if incident_filter not in [RECEIVED_FILTER, MODIFIED_FILTER]:  # Ensure it's one of the allowed filter values
                incident_filter = RECEIVED_FILTER  # or if not, force it to the default, RECEIVED_FILTER
            skip_unparsable_emails: bool = argToBoolean(params.get("skip_unparsable_emails", False))
            demisto.debug(f"{incident_filter=}, {skip_unparsable_emails=}")
            incidents = fetch_emails_as_incidents(client, last_run, incident_filter, skip_unparsable_emails)
            demisto.debug(f"Saving incidents with size {sys.getsizeof(incidents)}, len:{len(incidents)}")
            demisto.incidents(incidents)

        elif command == "send-mail":
            commands_res = send_email(client, args)
            return_results(commands_res)

        elif command == "ews-move-item-between-mailboxes":
            # Needs a client configured for each one of the mailboxes
            dest_params = params.copy()
            dest_params["default_target_mailbox"] = args.get("destination_mailbox")
            dest_client = get_client_from_params(dest_params)
            return_results(
                move_item_between_mailboxes(
                    src_client=client,
                    args=args,
                    dest_client=dest_client,
                )
            )

        # special outputs commands
        elif command in special_output_commands:
            demisto.results(special_output_commands[command](client, args))  # type: ignore[operator]

        elif command == "ews-auth-reset":
            return_results(reset_auth())

        # normal commands
        else:
            output = normal_commands[command](client, args)  # type: ignore[operator]
            if isinstance(output, tuple):  # Legacy, some commands return a tuple for return outputs
                return_outputs(*output)
            else:
                return_results(output)

    except Exception as e:
        demisto.error(f"got exception {e}")
        start_logging()
        debug_log = log_stream.getvalue()  # type: ignore[union-attr]
        error_message_simple = ""

        # Office365 regular maintenance case
        if isinstance(e, ErrorMailboxStoreUnavailable | ErrorMailboxMoveInProgress):
            log_message = (
                "Office365 is undergoing load balancing operations. As a result, the service is temporarily unavailable."
            )
            if demisto.command() == "fetch-incidents":
                demisto.info(log_message)
                demisto.incidents([])
                sys.exit(0)
            if is_test_module:
                demisto.results(log_message + " Please retry the instance configuration test.")
                sys.exit(0)
            error_message_simple = log_message + " Please retry your request."

        if isinstance(e, ConnectionError):
            error_message_simple = f"Could not connect to the server.\nAdditional information: {e!s}"
        else:
            if is_test_module and isinstance(e, MalformedResponseError):
                error_message_simple = "Got invalid response from the server.\n"

        # Legacy error handling
        if "Status code: 401" in debug_log:
            error_message_simple = "Got unauthorized from the server. "

        if "Status code: 503" in debug_log:
            error_message_simple = "Got timeout from the server. Probably the server is not reachable with the current settings. "

        if not error_message_simple:
            error_message = error_message_simple = str(e)
        else:
            error_message = error_message_simple + "\n" + str(e)

        stacktrace = traceback.format_exc()
        if stacktrace:
            error_message += "\nFull stacktrace:\n" + stacktrace

        if debug_log:
            error_message += "\nFull debug log:\n" + debug_log

        if demisto.command() == "fetch-incidents":
            raise
        if demisto.command() == "ews-search-mailbox" and isinstance(e, ValueError):
            return_error(
                message="Selected invalid field, please specify valid field name.",
                error=e,
            )
        if is_test_module:
            demisto.results(error_message_simple)
        else:
            demisto.results(
                {
                    "Type": entryTypes["error"],
                    "ContentsFormat": formats["text"],
                    "Contents": error_message_simple,
                }
            )
        demisto.error(f"{e.__class__.__name__}: {error_message}")
    finally:
        exchangelib_cleanup()
        if log_stream:
            try:
                logging.getLogger().removeHandler(log_handler)  # type: ignore
                log_stream.close()
            except Exception as ex:
                demisto.error(f"EWS: unexpected exception when trying to remove log handler: {ex}")


def process_main():
    """setup stdin to fd=0 so we can read from the server"""
    sys.stdin = os.fdopen(0, "r")
    sub_main()


def main():  # pragma: no cover
    # When running big queries, like 'ews-search-mailbox' the memory might not be freed by the garbage
    # collector. `separate_process` flag will run the integration on a separate process that will prevent
    # memory leakage.
    separate_process = demisto.params().get("separate_process", False)
    demisto.debug(f"Running as separate_process: {separate_process}")
    if separate_process:
        try:
            p = Process(target=process_main)
            p.start()
            p.join()
            demisto.debug("subprocess finished")
        except Exception as ex:
            demisto.error(f"Failed starting Process: {ex}")
    else:
        sub_main()


if __name__ in ("__main__", "__builtin__", "builtins"):
    main()