DedupBy

Source

import json
from typing import Any

import demistomock as demisto  # noqa: F401
from CommonServerPython import *  # noqa: F401


def demisto_get(obj: Any, path: Any) -> Any:
    r"""
    This is an extended function of demisto.get().
    The `path` argument parameter supports a syntax of path escaped with backslash
    in order to support a key including period characters.

    e.g.
       xxx
        + x.y.z
         + zzz

       -> path: xxx.x\.y\.z.zzz

    :param obj: The root node.
    :param path: The path to get values in the node.
    :return: The value(s) specified with `path` in the node.
    """

    def split_context_path(path: str) -> list[str]:
        """
        Get keys in order from the path which supports a syntax of path escaped with backslash.

        :param path: The path.
        :return: The keys whose escape characters are removed.
        """
        nodes = []
        node = []
        itr = iter(path)
        for c in itr:
            if c == "\\":
                try:
                    node.append(next(itr))
                except StopIteration:
                    node.append("\\")
            elif c == ".":
                nodes.append("".join(node))
                node = []
            else:
                node.append(c)
        nodes.append("".join(node))
        return nodes

    if not isinstance(obj, dict):
        return None

    for part in split_context_path(path):
        if obj and part in obj:
            obj = obj[part]
        else:
            return None
    return obj


class Key:
    """
    The custom key object class, which enables you to compare any types of data even in different types.
    This can be used for keys of dict.
    """

    def __init__(self, value: Any, path: Optional[str] = None) -> None:
        """
        Initialize the key.

        :param value: The value to set key, or the node from which to get the value if the `path` is given.
        :param path: The path to get values in the node.
        """
        self.__value = value if path is None else demisto_get(value, path)

    def __eq__(self, other: Any) -> bool:
        def __equals(obj1: Any, obj2: Any) -> bool:
            if type(obj1) != type(obj2):  # noqa: E721
                return False
            elif isinstance(obj1, dict):
                for k1, v1 in obj1.items():
                    if k1 not in obj2:
                        return False
                    if not __equals(v1, obj2[k1]):
                        return False
                return not (set(obj1.keys()) ^ set(obj2.keys()))
            elif isinstance(obj1, list):
                if len(obj1) != len(obj2):
                    return False
                return all(__equals(e1, e2) for e1, e2 in zip(obj1, obj2))
            else:
                return obj1 == obj2

        if not isinstance(other, Key):
            return False
        return __equals(self.__value, other.__value)

    def __hash__(self) -> int:
        """
        Generate hash using json.dumps for better performance.
        This replaces the recursive __get_hash_base approach with O(n) complexity.
        """
        try:
            # Use json.dumps with sort_keys for consistent hashing of dicts
            # This is O(n) instead of O(n*m) for the recursive approach
            json_str = json.dumps(self.__value, sort_keys=True, default=str)
            return hash((type(self.__value).__name__, json_str))
        except (TypeError, ValueError):
            # Fallback for non-JSON-serializable objects
            return hash((type(self.__value).__name__, str(self.__value)))


def main():
    try:
        args = assign_params(**demisto.args())
        if value := args.get("value", []):
            temp = {}
            if paths := argToList(args.get("keys")):
                for v in value:
                    k: tuple | Key = tuple(Key(v, path) for path in paths)
                    if k not in temp:
                        temp[k] = v
            else:
                for v in value:
                    k = Key(v)  # noqa: F812
                    if k not in temp:
                        temp[k] = v
            value = list(temp.values())

        return_results(value)
    except Exception as err:
        # Don't return an error by return_error() as this is transformer.
        raise DemistoException(str(err))


if __name__ in ("__builtin__", "builtins", "__main__"):
    main()

README

[
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.3"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.3"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.3"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.3"
  }
]

[
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.3"
  }
]

[
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "1.1.1.1",
    "SourceIP": "192.168.1.3"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.1"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.2"
  },
  {
    "DestinationIP": "2.2.2.2",
    "SourceIP": "192.168.1.3"
  }
]

[
  null,
  1,
  {
    "key": "value1"
  },
  2,
  0.5,
  0,
  0.5,
  "aaa",
  1,
  {
    "key": "value1"
  },
  null,
  "aaa",
  "ZZZ"
]

[
  null,
  1,
  {
    "key": "value1"
  },
  2,
  0.5,
  0,
  "aaa",
  "ZZZ"
]