AzureFeed

import re

import urllib3
from CommonServerPython import *

# disable insecure warnings
urllib3.disable_warnings()

INTEGRATION_NAME = "Azure"
AZUREJSON_URL = "https://www.microsoft.com/en-us/download/details.aspx?id=56519"  # disable-secrets-detection

ERROR_TYPE_TO_MESSAGE = {
    requests.ConnectionError: f"Connection error in the API call to {INTEGRATION_NAME}.\n",
    requests.exceptions.SSLError: f"Connection error in the API call to {INTEGRATION_NAME}.\n"
    f"Check your 'Trust any certificate' parameter.\n\n",
    requests.exceptions.HTTPError: f"Error issuing the request call to {INTEGRATION_NAME}.\n\n",
}


class Client(BaseClient):
    """Client to use in the Azure Feed integration. Overrides BaseClient.

    Args:
        regions_list (list): List of regions to filter.
        services_list (list): List of services to filter.
        insecure (bool): False if feed HTTPS server certificate should be verified, True otherwise.
        proxy (bool): False if feed HTTPS server certificate will not use proxies, True otherwise.
    """

    def __init__(
        self, regions_list: list, services_list: list, polling_timeout: int = 20, insecure: bool = False, proxy: bool = False
    ):
        super().__init__(base_url=AZUREJSON_URL, verify=not insecure, proxy=proxy)
        self.regions_list = regions_list
        self.services_list = services_list
        self._polling_timeout = polling_timeout

    @staticmethod
    def build_ip_indicator(azure_ip_address, **indicator_metadata) -> dict:
        """Creates an IP data dict.

        Args:
            azure_ip_address (str): IP extracted from Azure.
            **indicator_metadata (dict): Additional information related to the IP.

        Returns:
            Dict. IP data object.
        """
        if re.match(ipv4cidrRegex, azure_ip_address):
            type_ = FeedIndicatorType.CIDR

        elif re.match(ipv4Regex, azure_ip_address):
            type_ = FeedIndicatorType.IP

        elif re.match(ipv6cidrRegex, azure_ip_address):
            type_ = FeedIndicatorType.IPv6CIDR

        elif re.match(ipv6Regex, azure_ip_address):
            type_ = FeedIndicatorType.IPv6

        else:
            LOG(f"{INTEGRATION_NAME} - Unknown IP version: {azure_ip_address}")
            return {}

        ip_object = {
            "value": azure_ip_address,
            "type": type_,
        }
        ip_object.update(indicator_metadata)

        return ip_object

    def get_azure_download_link(self):
        """Extracts the download link for the file from the Azure url.

        Returns:
            str. The download link.
        """
        try:
            azure_url_response = self._http_request(
                method="GET",
                full_url=self._base_url,
                url_suffix="",
                headers={"User-Agent": "PANW-XSOAR"},
                stream=False,
                timeout=self._polling_timeout,
                resp_type="text",
                retries=4,
                status_list_to_retry=[403, 404],
            )
            download_link_search_regex = re.search(r"(https://download\.microsoft\.com/download/.+?\.json)", azure_url_response)
            download_link = download_link_search_regex.group(1) if download_link_search_regex else None
            if download_link is None:
                demisto.debug(f"azure response is: {azure_url_response}")
                raise RuntimeError(f"{INTEGRATION_NAME} - Download link not found")
            demisto.debug(f"download link: {download_link}")
            save_azure_download_link(download_link)

        except Exception as e:
            demisto.info(f"Error while fetching download link: {e}")
            download_link = load_azure_download_link()
            demisto.debug(f"Loaded cached download link: {download_link}")
            if not download_link:
                raise DemistoException("Unable to get download link.")

        return download_link

    def get_download_file_content_values(self, download_link: str) -> dict:
        """Create a request to receive file content from link.

        Args:
            download_link (str): Link to the desired Azure file.

        Returns:
            Dict. Content of values section in the Azure downloaded file.
        """
        file_download_response = self._http_request(
            method="GET", full_url=download_link, url_suffix="", stream=True, timeout=self._polling_timeout
        )

        return file_download_response.get("values")

    @staticmethod
    def extract_metadata_of_indicators_group(indicators_group_data: dict) -> dict:
        """Extracts metadata of an indicators group.

        Args:
            indicators_group_data (Dict): Indicator's group object from the Azure downloaded file.

        Returns:
            Dict. Indicators group metadata.
        """
        indicator_metadata = {}

        indicator_metadata["id"] = indicators_group_data.get("id")
        indicator_metadata["name"] = indicators_group_data.get("name")
        indicator_properties = indicators_group_data.get("properties")

        if not indicator_properties:
            LOG(f'{INTEGRATION_NAME} - no properties for indicators group {indicator_metadata["name"]}')
            return {}

        indicator_metadata["region"] = indicator_properties.get("region")
        indicator_metadata["platform"] = indicator_properties.get("platform")
        # The first part of the ID is the service name by default. For example {ID: "a.b", serviceName: "a"}.
        indicator_metadata["system_service"] = (
            indicator_properties.get("systemService") or str(indicator_metadata.get("id", ".")).split(".")[0]
        )
        indicator_metadata["address_prefixes"] = indicator_properties.get("addressPrefixes", [])

        return indicator_metadata

    @staticmethod
    def filter_and_aggregate_values(address_list: list) -> list:
        """For each indicator value from the given list we aggregate the all the different keys found.

        Args:
            address_list (List): list of indicator objects containing objects with duplicate values.

        Returns:
            List. List of filtered indicator objects (no indicator value appear twice) and aggregated data
        """
        indicator_objects: dict = {}
        for item_to_search in address_list:
            current_value = item_to_search.get("value")
            ind_obj = indicator_objects.get(current_value)
            if ind_obj:
                indicator_objects[current_value].update(item_to_search)
            else:
                indicator_objects[current_value] = item_to_search

        return list(indicator_objects.values())

    def extract_indicators_from_values_dict(self, values_from_file: dict) -> list:
        """Builds a list of all IP indicators in the input dict.

        Args:
            values_from_file (Dict): The values object from the Azure downloaded file.

        Returns:
            list. All indicators that match the filtering options.
        """
        results = []

        if values_from_file is None:
            LOG(f"{INTEGRATION_NAME} - No values in JSON response")
            return []

        for indicators_group in values_from_file:
            demisto.debug(f'{INTEGRATION_NAME} - Extracting value: {indicators_group.get("id")}')

            indicator_metadata = self.extract_metadata_of_indicators_group(indicators_group)
            if not indicator_metadata:
                continue

            is_region_not_in_filter = "All" not in self.regions_list and indicator_metadata["region"] not in self.regions_list
            is_service_not_in_filter = (
                "All" not in self.services_list and indicator_metadata["system_service"] not in self.services_list
            )

            if is_region_not_in_filter or is_service_not_in_filter:
                continue

            for address in indicator_metadata["address_prefixes"]:
                results.append(
                    self.build_ip_indicator(
                        address,
                        azure_name=indicator_metadata["name"],
                        azure_id=indicator_metadata["id"],
                        azure_region=indicator_metadata["region"],
                        azure_platform=indicator_metadata["platform"],
                        azure_system_service=indicator_metadata["system_service"],
                    )
                )
        return self.filter_and_aggregate_values(results)

    def build_iterator(self) -> list:
        """Retrieves all entries from the feed.
        Returns:
            A list of objects, containing the indicators.
        """
        try:
            download_link = self.get_azure_download_link()
            values_from_file = self.get_download_file_content_values(download_link)
            return self.extract_indicators_from_values_dict(values_from_file)

        except (requests.exceptions.SSLError, requests.ConnectionError, requests.exceptions.HTTPError) as err:
            demisto.debug(str(err))
            raise Exception(ERROR_TYPE_TO_MESSAGE[err.__class__] + str(err))

        except RuntimeError as err:
            demisto.debug(str(err))
            raise RuntimeError("Could not fetch download link from Azure")

        except ValueError as err:
            demisto.debug(str(err))
            raise ValueError(f"Could not parse returned data to Json. \n\nError message: {err}")


def load_azure_download_link() -> Optional[str]:
    """Loads the download link for the file from the server.

    Returns:
        str. The download link.
    """
    return demisto.getIntegrationContext().get("azure_download_link")


def save_azure_download_link(link: str):
    """Saves the download link for the file to the server."""
    demisto.setIntegrationContext(demisto.getIntegrationContext() | {"azure_download_link": link})


def test_module(client: Client) -> tuple[str, dict, dict]:
    """Test the ability to fetch Azure file.
    Args:
        client: Client object.
    Returns:
        str. ok for success, relevant error string otherwise.
    """
    try:
        if "All" in client.regions_list and len(client.regions_list) >= 2:
            err_msg = "ConfigurationError: You may not select additional regions if you selected 'All'"
            return_error(err_msg)

        if "All" in client.services_list and len(client.services_list) >= 2:
            err_msg = "ConfigurationError: You may not select additional services if you selected 'All'"
            return_error(err_msg)

        download_link = client.get_azure_download_link()
        client.get_download_file_content_values(download_link)

    except (requests.exceptions.SSLError, requests.ConnectionError, requests.exceptions.HTTPError) as err:
        demisto.debug(str(err))
        raise Exception(ERROR_TYPE_TO_MESSAGE[err.__class__] + str(err))

    return "ok", {}, {}


def get_indicators_command(
    client: Client, feedTags: list, tlp_color: str | None, enrichment_excluded: bool = False
) -> tuple[str, dict, dict]:
    """Retrieves indicators from the feed to the war-room.

    Args:
        client (Client): Client object configured according to instance arguments.
        feedTags (list): The indicator tags.
        tlp_color (str): Traffic Light Protocol color

    Returns:
        Tuple of:
            str. Information to be printed to war room.
            Dict. The raw data of the indicators.
    """
    limit = int(demisto.args().get("limit")) if "limit" in demisto.args() else 10
    indicators, raw_response = fetch_indicators_command(client, feedTags, tlp_color, limit, enrichment_excluded)

    human_readable = tableToMarkdown("Indicators from Azure Feed:", indicators, headers=["value", "type"], removeNull=True)

    return human_readable, {}, {"raw_response": raw_response}


def fetch_indicators_command(
    client: Client, feedTags: list, tlp_color: str | None, limit: int = -1, enrichment_excluded: bool = False
) -> tuple[list[dict], list]:
    """Fetches indicators from the feed to the indicators tab.
    Args:
        client (Client): Client object configured according to instance arguments.
        limit (int): Maximum number of indicators to return.
        feedTags (list): Indicator tags
        tlp_color (str): Traffic Light Protocol color
    Returns:
        Tuple of:
            str. Information to be printed to war room.
            Dict. Data to be entered to context.
            Dict. The raw data of the indicators.
    """
    iterator = client.build_iterator()
    indicators = []
    raw_response = []

    if limit != -1:
        iterator = iterator[:limit]

    for indicator in iterator:
        indicator_obj = {
            "value": indicator["value"],
            "type": indicator["type"],
            "fields": {
                "region": indicator.get("azure_region"),
                "service": indicator.get("azure_system_service"),
                "tags": feedTags,
            },
            "rawJSON": indicator,
        }

        if tlp_color:
            indicator_obj["fields"]["trafficlightprotocol"] = tlp_color

        if enrichment_excluded:
            indicator_obj["enrichmentExcluded"] = enrichment_excluded

        indicators.append(indicator_obj)
        raw_response.append(indicator)

    return indicators, raw_response


def main():
    """
    PARSE AND VALIDATE INTEGRATION PARAMS
    """
    regions_list = argToList(demisto.params().get("regions"))
    if not regions_list:
        regions_list = ["All"]

    services_list = argToList(demisto.params().get("services"))
    if not services_list:
        services_list = ["All"]

    feedTags = argToList(demisto.params().get("feedTags"))
    tlp_color = demisto.params().get("tlp_color")
    enrichment_excluded = demisto.params().get("enrichmentExcluded", False)

    polling_arg = demisto.params().get("polling_timeout", "")
    polling_timeout = int(polling_arg) if polling_arg.isdigit() else 20
    insecure = demisto.params().get("insecure", False)
    proxy = demisto.params().get("proxy", False)

    command = demisto.command()
    demisto.info(f"Command being called is {command}")

    try:
        client = Client(regions_list, services_list, polling_timeout, insecure, proxy)
        if command == "test-module":
            return_outputs(*test_module(client))
        elif command == "azure-get-indicators":
            if feedTags:
                feedTags["tags"] = feedTags
            return_outputs(*get_indicators_command(client, feedTags, tlp_color))
        elif command == "fetch-indicators":
            indicators, _ = fetch_indicators_command(client, feedTags, tlp_color, enrichment_excluded=enrichment_excluded)
            for single_batch in batch(indicators, batch_size=2000):
                demisto.createIndicators(single_batch)

        else:
            raise NotImplementedError(f"Command {command} is not implemented.")

    except Exception:
        raise


if __name__ in ["__main__", "builtin", "builtins"]:
    main()