Moloch Deprecated

Deprecated. Use Arkime instead.

Network Security · Moloch (Deprecated)

Details

IDMoloch
ProviderOpen Source
CategoryNetwork Security
From Version5.0.0
Supported ModulesAgentix

README

Overview

Use the Moloch integration to store and index network traffic in standard PCAP format.

This integration was integrated and tested with Moloch v1.5.1.

 

Configure Moloch on Cortex XSOAR

  1. Navigate to Settings > Integrations > Servers & Services.
  2. Search for Moloch.
  3. Click Add instance to create and configure a new integration instance.
    • Name: a textual name for the integration instance.
    • Server URL (e.g. https://192.168.0.1)
    • Username
    • Trust any certificate (not secure)
  4. Click Test to validate the URLs, token, and connection.

 

Commands

You can execute these commands from the Cortex XSOAR CLI, as part of an automation, or in a playbook.
After you successfully execute a command, a DBot message appears in the War Room with the command details.

  1. Retrieve connections data in JSON: moloch_connections_json
  2. Retrieve connections data in CSV: moloch_connections_csv
  3. Return a list of files: moloch_files_json
  4. Retrieve session data in JSON: moloch_sessions_json
  5. Retrieve session data in CSV: moloch_sessions_csv
  6. Retrieve session data in PCAP: moloch_sessions_pcap
  7. Retrieve Spigraph data in JSON: moloch_spigraph_json
  8. Retrieve Spiview data in JSON: moloch_spiview_json
  9. Retrieve unique data for a field in JSON: moloch_unique_json

 

1. Retrieve connections data in JSON

Retrieve the connections data in JSON format.

Base Command

moloch_connections_json

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
dstField The source database field name (Default: a2) Optional
expression The expression string Optional
iDisplayLength Number of items to return (Default: 5000, Max: 2000000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length The number of items to return (Default: 5000, Max: 2000000) Optional
srcField The source database field name (Default: a1) Optional
start The entry to start from (Default: 0) Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_connections_json startTime="2014/02/26 10:27:57"

Human Readable Output
{
    "health": {
        "_timeStamp": 1534839251551,
        "active_primary_shards": 380,
        "active_shards": 380,
        "active_shards_percent_as_number": 100,
        "cluster_name": "Moloch",
        "delayed_unassigned_shards": 0,
        "initializing_shards": 0,
        "molochDbVersion": 51,
        "number_of_data_nodes": 1,
        "number_of_in_flight_fetch": 0,
        "number_of_nodes": 1,
        "number_of_pending_tasks": 0,
        "relocating_shards": 0,
        "status": "green",
        "task_max_waiting_in_queue_millis": 0,
        "timed_out": false,
        "unassigned_shards": 0,
        "version": "5.6.4"
    },
    "links": [
        {
            "by": 136284,
            "db": 121356,
            "node": {
                "demo": 1
            },
            "pa": 1866,
            "source": 0,
            "target": 1,
            "value": 4
        },
        {
            "by": 8999,
            "db": 8231,
            "node": {
                "demo": 1,
                "ip-10-97-23-168": 1
            },
            "pa": 96,
            "source": 2,
            "target": 3,
            "value": 4
        }
    ],
    "nodes": [
        {
            "by": 136284,
            "cnt": 1,
            "db": 121356,
            "id": "1.1.1.1",
            "pa": 1866,
            "pos": 0,
            "sessions": 4,
            "type": 1
        },
        {
            "by": 136284,
            "cnt": 1,
            "db": 121356,
            "id": "2.2.2.2",
            "pa": 1866,
            "pos": 1,
            "sessions": 4,
            "type": 2
        }
    ],
    "recordsFiltered": 145724
}

 

2. Retrieve connections data in CSV: moloch_connections_csv

Retrieve the connections data in CSV format.

Base Command

moloch_connections_csv

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
dstField The source database field name (Default: a2) Optional
expression The expression string Optional
iDisplayLength The number of items to return (Default: 5000, Max: 2000000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length The number of items to return (Default: 5000, Max: 2000000) Optional
srcField The source database field name (Default: a1) Optional
start The entry to start at (Default: 0) Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_connections_csv date="-1"

Human Readable Output

screen shot 2018-08-21 at 11 32 06

 

3. Return a list of files

Return a list of files in the Moloch database.

Base Command

moloch_files_json

Input
Argument Name Description Required
iDisplayLength The number of items to return (Default: 500, Max: 10000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length The number of items to return (Default: 500, Max: 10000) Optional
start The entry to start at (Default: 0) Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_files_json length="10"

Human Readable Output
{
    "data": [
        {
            "filesize": 15819,
            "first": 1273057060,
            "id": "demo-1",
            "locked": 1,
            "name": "/moloch/1filtered.cap",
            "node": "demo",
            "num": 1
        },
        {
            "filesize": 2514,
            "first": 1249662076,
            "id": "demo-2",
            "locked": 1,
            "name": "/moloch/20090807_portal_prod_io0_01.cap",
            "node": "demo",
            "num": 2
        }
    ],
    "recordsFiltered": 434,
    "recordsTotal": 434
}

 

4. Retrieve session data in JSON

Retrieve the session data in JSON format.

Base Command

moloch_sessions_json

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
expression The expression string Optional
facets Also include the aggregation information for maps and time graphs Optional
iDisplayLength The number of items to return (Default: 100, Max: 2000000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length The number of items to return (Default: 100, Max: 2000000) Optional
start The entry to start at (Default: 0) Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_sessions_json stopTime="2014/02/26 11:27:57"

Human Readable Output

image

 

5. Retrieve session data in CSV

Retrieve the session data in CSV format.

Base Command

moloch_sessions_csv

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
expression The expression string Optional
facets Also include the aggregation information for maps and time graphs Optional
iDisplayLength The number of items to return (Default: 100, Max: 2000000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length the number of items to return (Default: 100, Max: 2000000) Optional
start The entry to start at (Default: 0) Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_sessions_csv

 
Human Readable Output

image

 

6. Retrieve raw session data in PCAP

Retrieve the raw session data in PCAP format.

Base Command

moloch_sessions_pcap

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
expression The expression string, used if ids not set Optional
ids The list of ids to return Optional
iDisplayLength The number of items to return (Default: 100, Max: 2000000) Optional
iDisplayStart The entry to start from (Default: 0) Optional
length The number of items to return (Default: 100, Max: 2000000) Optional
segments When set return linked segments Optional
start The entry to start at (Default: 0) Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

 
Command Example

!moloch_sessions_pcap startTime="1520542248" stopTime="1533329500"

 
Human Readable Output

image

 

7. Retrieve Spigraph data in JSON

Retrieve the Spigraph data in JSON format.

Base Command

moloch_spigraph_json

Input
Argument Name Description Required
date The number of hours to return data for (-1 returns all data) Optional
expression The expression string Optional
field The database field name to spigraph on Optional
size The number of unique values to return Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

 
Command Example

!moloch_spigraph_json startTime=1520542248 stopTime=1533329500

 
Human Readable Output

image
image

 

8. Retrieve Spiview data in JSON

Retrieve the Spiview data in JSON format.

Base Command

moloch_spiview_json

Input
Argument Name Description Required
date The number of hours of data to return (-1 returns all data) Optional
expression The expression string Optional
spi A comma-separated list of fields to return data for Optional
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When this argument is used, the entire session must be within the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_spiview_json startTime=1520542248 stopTime=1533329500

Human Readable Output

image

 

9. Retrieve unique data for a field in JSON

Retrieve unique data for a specified field in JSON format.

Base Command

moloch_unique_json

Input
Argument Name Description Required
date The number of hours of data to return (-1 returns all data) Optional
expression The expression string Optional
field The database field name to unique on Required
startTime If the date parameter is not set, this is the start time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
stopTime If the date parameter is not set, this is the stop time of the date to return. If an integer is used (the number of seconds since Unix EPOC), otherwise parsed using JavaScript Date parser. Usage example: !moloch_sessions_json startTime="2014/02/26 10:27:57". For more  examples see here. Optional
strictly When set the entire session must be inside the date range to be observed, otherwise if it overlaps it is displayed Optional
view The view name to apply before the expression Optional
 
Context Output

There is no context output for this command.

Command Example

!moloch_unique_json date="-1" field="https.status"

Configuration parameters

  • server — Server URL (e.g. https://192.168.0.1) (required)
  • credentials — Username (required)
  • insecure — Trust any certificate (not secure)

Commands (9)

  • moloch_connections_csv

    Retrieve the connections data in csv format

  • moloch_connections_json

    Retrieve the connections data in json format

  • moloch_files_json

    Return a list of files that moloch knows about

  • moloch_sessions_csv

    Retrieve the session data in csv format

  • moloch_sessions_json

    Retrieve the session data in json format

  • moloch_sessions_pcap

    Retrieve the raw session data in pcap format

  • moloch_spigraph_json

    Retrieve the spigraph data in json format

  • moloch_spiview_json

    Retrieve the spiview data in json format

  • moloch_unique_json

    Retrieve the unique data for a field in json format

Source

Moloch.js Moloch.yml
Download History Raw 
var username = params.credentials.identifier;
var password = params.credentials.password;
var server = params.server;
var insecure = params.insecure;

var sendRequest = function(url, queryName) {
    var res = http(
            server + '/' + url,
            {
                Method: 'GET',
                Username: username,
                Password: password,
                UseAuthDigest: true,
            },
            insecure
        );
    if (res.StatusCode < 200 || res.StatusCode >= 300) {
        throw 'Failed to ' + queryName + ', request status code: ' + res.StatusCode + ' and Body: ' + res.Body + '.';
    }
    return res;
}

var pcap_command = 'moloch_sessions_pcap';
var unique_command = 'moloch_unique_json';

// Fields to get from the db while sending moloch_sessions_json requests
var other_fields = "ipProtocol";
var http_fields = "http.statuscode,http.method";
var src_fields = "srcPackets,srcBytes,srcDataBytes";
var dst_fields = "dstPackets,dstBytes,dstDataBytes";
var fields_to_req = "&fields={0},{1},{2},{3}&expression=http.statuscode==EXISTS!".format(http_fields, src_fields, dst_fields,other_fields);

var urlDict = {
    'moloch_connections_json': 'connections.json',
    'moloch_connections_csv': 'connections.csv',
    'moloch_files_json': 'file/list',
    'moloch_sessions_json': 'sessions.json',
    'moloch_sessions_csv': 'sessions.csv',
    'moloch_sessions_pcap': 'sessions.pcap',
    'moloch_spigraph_json': 'spigraph.json',
    'moloch_spiview_json': 'spiview.json',
    'moloch_unique_json': 'unique.txt'
}

var prToName = {
    1: 'icmp',
    6: 'tcp',
    17: 'udp',
    47: 'gre',
    58: 'icmpv6'
}

var jsonResponses = ['moloch_connections_json', 'moloch_files_json', 'moloch_sessions_json', 'moloch_spigraph_json', 'moloch_spiview_json'];
var csvResponses = ['moloch_connections_csv', 'moloch_sessions_csv'];
var pcapResponses = ['moloch_sessions_pcap'];

var parseSessions = function(response) {
    var outputs = [];
    var contexts = [];
    for (var i = 0, objects = response.data; i < objects.length; i++) {
        var output = {};

        output['ID'] = objects[i].id;
        output['Index'] = objects[i].index;
        output['Protocol'] = objects[i].ipProtocol in prToName ? prToName[objects[i].ipProtocol] : objects[i].ipProtocol;
        output['Start Time'] = args.startTime;
        output['End Time'] = args.stopTime;

        output['Source IP'] = objects[i].srcIp;
        output['Source Port'] = objects[i].srcPort;
        output['Source Packets'] = objects[i].srcPackets;
        output['Source Bytes'] = objects[i].srcBytes;
        output['Source Databytes'] = objects[i].srcDataBytes;

        output['Destination IP'] = objects[i].dstIp;
        output['Destination Port'] = objects[i].dstPort;
        output['Destination Packets'] = objects[i].dstPackets;
        output['Destination Bytes'] = objects[i].dstBytes;
        output['Destination Databytes'] = objects[i].dstDataBytes;

        output['HTTP method'] = objects[i].http.method;
        output['HTTP Status code'] = objects[i].http.statuscode;

        outputs.push(output);
        contexts.push(convertKeysToPascalCase(output));
    }

    entry = {
        Type: entryTypes.note,
        Contents: null,
        ContentsFormat: formats.json,
        EntryContext: {'Moloch.Sessions': contexts},
        HumanReadable: tableToMarkdown('Moloch Sessions Search', outputs)
    };
    return entry;
}

switch (command) {
    case 'test-module':
        if (sendRequest(urlDict['moloch_connections_json'], 'Test').Body) {
            return 'ok';
        }

        return 'not cool';

    default:
        var currentTime = new Date();

        if (!args.date && command != pcap_command && command != unique_command) {
            args.date = -1;
        }

        url = urlDict[command] + encodeToURLQuery(args);

        var res = sendRequest(url, urlDict[command]);
        if (!res || !res.Body) {
            return 'No data was found in Moloch';
        }

        if (command == unique_command) {
            return res.Body;
        } else if (jsonResponses.indexOf(command) !== -1) {
            if (command == 'moloch_sessions_json') {
                return parseSessions(JSON.parse(res.Body));
            } else {
                return JSON.parse(res.Body);
            }
        } else if (csvResponses.indexOf(command) !== -1) {
            return {Type: 9, FileID: saveFile(res.Bytes), File: command + encodeToURLQuery(args).substr(1) + '_at_' + currentTime.getTime() + '.csv',  Contents: 'we must have contents for an entry'};
        } else if (pcapResponses.indexOf(command) !== -1) {
            return {Type: 9, FileID: saveFile(res.Bytes), File: command + encodeToURLQuery(args).substr(1) + '_at_' + currentTime.getTime() + '.pcap',  Contents: 'we must have contents for an entry'};
        }

        return res;
}