UnEscapeURLs

Extract URLs redirected by security tools like Proofpoint. Changes https://urldefense.proofpoint.com/v2/url?u=https-3A__example.com_something.html -> https://example.com/something.html Also, un-escape URLs that are escaped for safety with formats like hxxps://www[.]demisto[.]com.

Type

javascript

Pack

CommonScripts

Source

function format_url(input) {
    // Check if it is a Microsoft ATP Safe Link
    var atp_link_re = /(https:\/\/\w*|\w*)\.safelinks\.protection\.outlook\.com\/.*\?url=/;
    if (atp_link_re.test(input)) {
        var index_of_param_section = 1;
        var url_as_parts = input.split("?");
        var url_params_section = null;
        if (index_of_param_section < url_as_parts.length) {
            url_params_section = url_as_parts[index_of_param_section];
        }
        var url_params = null;
        if (url_params_section !== null) {
            url_params = url_params_section.split("&");
        }
        var encoded_url = null;
        if (url_params !== null) {
            for (i = 0; i < url_params.length; i++) {
                param_name = url_params[i].split("=");
                if (param_name[0] == "url") encoded_url = param_name[1];
            }
        }
        if (encoded_url === null) {
            error_msg = "Could not decode ATP Safe Link. Returning original URL.";
            logInfo(error_msg);
            return input;
        }
        decoded_url = decodeURIComponent(encoded_url);
        return decoded_url;
    }
    // Check if it is a Proofpoint Protected URL, including gov cloud
    var PROOFPOINT_PREFIXES = ['https://urldefense.proofpoint.com/v1/url?u=', 'https://urldefense.proofpoint.com/v2/url?u=', 'https://urldefense.com/v3/__', 'https://urldefense.us/v3/__'];
    var v = 0;
    if (input.indexOf(PROOFPOINT_PREFIXES[0]) === 0) {
        v = 1;
    } else if (input.indexOf(PROOFPOINT_PREFIXES[1]) === 0) {
        v = 2;
    } else if (input.indexOf(PROOFPOINT_PREFIXES[2]) === 0) {
        v = 3;
    } else if (input.indexOf(PROOFPOINT_PREFIXES[3]) === 0) {
        v = 4;
    }
    // Not Proofpoint so just un-escape
    if (v === 0) {
        var url = input.replace(/\[\.\]/g, '.')
      if (input.toLowerCase().startsWith('https') || input.toLowerCase().startsWith('hxxps') || input.toLowerCase().startsWith('meows')) {
          //Normalize: 1) [.] --> . 2) hxxps --> https 3) meows --> https 4) &amp --> & 5) https:\\ --> https://
          url = 'https' + url.slice(5, url.length)
          url = url.replace(/&amp;/g, '&').replace(/https:\\\\/g, 'https://').replace(/https:\\/g, 'https://')
            // Normalize the URL with https prefix
            if (url.startsWith('https:/') && !url.startsWith('https://')) {
                url = url.replace('https:/', 'https://')
            }
            if (url.search(/https:/i) === 0 && url.search(/https:\/\//i) !== 0) {
                url = url.replace(/https:/i, 'https://')
            }
        }
      else if (input.toLowerCase().startsWith('http') || input.toLowerCase().startsWith('hxxp') || input.toLowerCase().startsWith('meow')) {
          //Normalize: 1) [.] --> . 2) hxxp --> http 3) meow --> http 4) &amp --> & 5) http:\\ --> http://
          url = 'http' + url.slice(4, url.length)
          url = url.replace(/&amp;/g, '&').replace(/http:\\\\/g, 'http://').replace(/http:\\/g, 'http://')

            // Normalize the URL with http prefix
            if (url.startsWith('http:/') && !url.startsWith('http://')) {
                url = url.replace('http:/', 'http://')
            }
            if (url.search(/http:/i) === 0 && url.search(/http:\/\//i) !== 0) {
                url = url.replace(/http:/i, 'http://')
            }
        }
        if (url.search(/http/i) !== 0 && url.search(/ftp/i) !== 0) {
            return 'http://' + url;
        }
        return url;
    }
    // If proofpoint type v3
    if (v === 3 || v === 4) {
        var clean = input.substr(PROOFPOINT_PREFIXES[v - 1].length);
        // The v3 URL ends with '__;' (double underscore + semicolon)
        var closeIndex = clean.indexOf('__;');
        if (closeIndex > -1) {
            clean = clean.substr(0, closeIndex);
        }
        var url_reg = /((http)s?:)(\/{1,2})(.*)/g;
        var clean_fixed = clean.replace(url_reg, '$1//$4');
        return clean_fixed;
    }

    // If from Proofpoint then no need to un-escape or normalize
    var re = /&.*$/;
    var at_sign_reg = /%40/g;
    var ampersand_reg = /%26/g;
    var hash_reg = /%23/g;
    var equals_reg = /%3D/g;
    var clean = input.substr(PROOFPOINT_PREFIXES[v - 1].length).replace(/-/g, '%').replace(/_/g, '/').replace('%3A//', '://').replace(re, '').replace('%3F', '?').replace('%3D', '=').replace(at_sign_reg, '@').replace(ampersand_reg, '&').replace(hash_reg, '#').replace(equals_reg, '=');
    try {
        clean = decodeURI(clean);
    } catch (e) {
    // Ignore if can't decodeURI
    }

    return clean;
}

function format_url_list(url_list) {
    var len = url_list.length;
    var formatted_urls = new Array(len);
    url_list.forEach(function(the_url, index) {
        formatted_urls[index] = format_url(the_url.trim());
    });
    return formatted_urls;
}

var urls;
// It is assumed that args.input is a string
var unformatted_urls = argToList(args.input);
urls = format_url_list(unformatted_urls);
return urls;

README