# coding=utf-8 from __future__ import print_function import pytest import demistomock as demisto from CommonServerPython import entryTypes from ParseEmailFiles import MsOxMessage, main, convert_to_unicode, unfold, handle_msg, get_msg_mail_format, \ data_to_md, create_headers_map, DataModel, handle_eml, parse_nesting_level def exec_command_for_file( file_path, info="RFC 822 mail text, with CRLF line terminators", file_name=None, file_type="", ): """ Return a executeCommand function which will return the passed path as an entry to the call 'getFilePath' Arguments: file_path {string} -- file name of file residing in test_data dir Raises: ValueError: if call with differed name from getFilePath or getEntry Returns: [function] -- function to be used for mocking """ if not file_name: file_name = file_path path = 'test_data/' + file_path def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': path, 'name': file_name } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': info, 'type': file_type } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) return executeCommand def test_msg_html_with_attachments(): msg = MsOxMessage('test_data/html_attachment.msg') assert msg is not None msg_dict = msg.as_dict(max_depth=2) assert 'This is an html email' in msg_dict['Text'] attachments_list = msg.get_all_attachments() assert len(attachments_list) == 1 attach = attachments_list[0] assert attach.AttachFilename == 'dummy-attachment.txt' assert attach.AttachMimeTag == 'text/plain' assert attach.data == 'This is a text attachment' def test_msg_utf_encoded_subject(): msg = MsOxMessage('test_data/utf_subject.msg') assert msg is not None msg_dict = msg.as_dict(max_depth=2) # we test that subject which has utf-8 encoding (in the middle) is actually decoded assert '?utf-8' in msg_dict['HeadersMap']['Subject'] subj = msg_dict['Subject'] assert 'TESTING' in subj and '?utf-8' not in subj def test_eml_smtp_type(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/smtp_email_type.eml', 'name': 'smtp_email_type.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'SMTP mail, UTF-8 Unicode text, with CRLF terminators' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' # assert demisto.executeCommand('getFilePath', {})[0]['Type'] == entryTypes['note'] main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Test Smtp Email' # this is a test for another version of a multipart signed eml file def test_smime2(mocker): multipart_sigened = 'multipart/signed; protocol="application/pkcs7-signature";, ASCII text, with CRLF line terminators' mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smime2.p7m', info=multipart_sigened)) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() # assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] # assert results[0]['EntryContext']['Email']['Subject'] == 'Testing signed multipart email' assert results[0]['EntryContext']['Email']['Subject'] == 'Testing signed multipart email' def test_handle_eml_parses_correct_message_id(): """ Given: - eml file When: - parsing eml file into email data. Then: - Validate that correct 'Message-ID' case sensitive is in 'HeadersMap' dict. Must be 'Message-ID' case sensitive. """ email_data, _ = handle_eml(file_path='test_data/invalid_message_id.eml') assert 'Message-ID' in email_data['HeadersMap'] def test_eml_contains_eml(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/Fwd_test-inner_attachment_eml.eml', 'name': 'Fwd_test-inner_attachment_eml.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'news or mail text, ASCII text' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 5 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][0]['Subject'] == 'Fwd: test - inner attachment eml' assert 'ArcSight_ESM_fixes.yml' in results[0]['EntryContext']['Email'][0]['Attachments'] assert 'ArcSight_ESM_fixes.yml' in results[0]['EntryContext']['Email'][0]['AttachmentsData'][0]['Name'] assert 'test - inner attachment eml.eml' in results[0]['EntryContext']['Email'][0]['Attachments'] assert 'test - inner attachment eml.eml' in results[0]['EntryContext']['Email'][0]['AttachmentsData'][1]['Name'] assert results[0]['EntryContext']['Email'][0]['Depth'] == 0 assert results[0]['EntryContext']['Email'][1]["Subject"] == 'test - inner attachment eml' assert 'CS Training 2019 - EWS.pptx' in results[0]['EntryContext']['Email'][1]["Attachments"] assert 'CS Training 2019 - EWS.pptx' in results[0]['EntryContext']['Email'][1]["AttachmentsData"][0]['Name'] assert results[0]['EntryContext']['Email'][1]['Depth'] == 1 def test_eml_contains_msg(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/DONT_OPEN-MALICIOUS.eml', 'name': 'DONT_OPEN-MALICIOUS.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'news or mail text, ASCII text' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 3 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][0]['Subject'] == 'DONT OPEN - MALICIOS' assert results[0]['EntryContext']['Email'][0]['Depth'] == 0 assert 'Attacker+email+.msg' in results[0]['EntryContext']['Email'][0]['Attachments'] assert 'Attacker+email+.msg' in results[0]['EntryContext']['Email'][0]['AttachmentsData'][0]['Name'] assert results[0]['EntryContext']['Email'][1]["Subject"] == 'Attacker email' assert results[0]['EntryContext']['Email'][1]['Depth'] == 1 def test_eml_contains_eml_depth(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/Fwd_test-inner_attachment_eml.eml', 'name': 'Fwd_test-inner_attachment_eml.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'news or mail text, ASCII text' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 3 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Fwd: test - inner attachment eml' assert 'ArcSight_ESM_fixes.yml' in results[0]['EntryContext']['Email']['Attachments'] assert 'ArcSight_ESM_fixes.yml' in results[0]['EntryContext']['Email']['AttachmentsData'][0]['Name'] assert 'test - inner attachment eml.eml' in results[0]['EntryContext']['Email']['Attachments'] assert 'test - inner attachment eml.eml' in results[0]['EntryContext']['Email']['AttachmentsData'][1]['Name'] assert isinstance(results[0]['EntryContext']['Email'], dict) assert results[0]['EntryContext']['Email']['Depth'] == 0 def test_eml_utf_text(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/utf_8_email.eml', 'name': 'utf_8_email.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'UTF-8 Unicode text, with very long lines, with CRLF line terminators' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Test UTF Email' def test_eml_utf_text_with_bom(mocker): '''Scenario: Parse an eml file that is UTF-8 Unicode (with BOM) text Given - A UTF-8 encoded eml file with BOM When - Executing ParseEmailFiles automation on the uploaded eml file Then - Ensure eml email file is properly parsed ''' def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/utf_8_with_bom.eml', 'name': 'utf_8_with_bom.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'RFC 822 mail text, UTF-8 Unicode (with BOM) text, ' 'with very long lines, with CRLF line terminators' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Test UTF Email' def test_email_with_special_character(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/email_with_special_char_bytes.eml', 'name': 'email_with_special_char_bytes.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'RFC 822 mail text, ISO-8859 text, with very long lines, with CRLF line terminators' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Hello dear friend' @pytest.mark.parametrize('encoded_subject, decoded_subject', [ ( '[TESTING] =?utf-8?q?=F0=9F=94=92_=E2=9C=94_Votre_colis_est_disponible_chez_votre_co?= ' '=?utf-8?q?mmer=C3=A7ant_Pickup_!?=', # noqa E501 '[TESTING]\xf0\x9f\x94\x92 \xe2\x9c\x94 Votre colis est disponible chez votre commer\xc3\xa7ant Pickup !' ), ( 'This =?UTF-8?B?VGVzdMKu?= passes', 'This Test® passes' ), ( '=?utf-8?B?44CQ?= =?utf-8?B?4pGg?=', # test case: double utf-8 byte encoded '\xe3\x80\x90\xe2\x91\xa0' # 【① ), ( '=?iso-2022-jp?B?GyRCJWEhPCVrLSEkSHxxGyhC?= ' '=?iso-2022-jp?B?GyRCRnxLXDhsSjg7eiQsST08KCQ1JGwkSiQkSjg7eiROJUYlOSVIGyhC?=', 'メール�と�日本語文字が表示されない文字のテスト' ), ( '=?UTF-8?Q?TEST_UNDERSCORE?=', 'TEST UNDERSCORE' ), # ( # 'This is test =?iso-2022-jp?B?GyRCJWEhPCVrLSEkSHxxGyhC?= ' # '=?iso-2022-jp?B?GyRCRnxLXDhsSjg7eiQsST08KCQ1JGwkSiQkSjg7eiROJUYlOSVIGyhC?=', # 'This is test メール�と�日本語文字が表示されない文字のテスト' # ) # ( 'Test =?UTF-8?Q?Seguran=C3=A7a=20?=da =?UTF-8?Q?Informa=C3=A7=C3=A3o?=', # 'Test Departamento de Teste' # ) # This test should pass, it extend the case of This example "=?UTF-8?B?VGVzdMKu?= passes" and include multiple # encoding parts. # **please DO NOT delete the commented tests**. # they have been disabled in attempt to fix issue no. 40877, and they may be needed for a better solution in the future. ]) def test_utf_subject_convert(encoded_subject, decoded_subject): decoded = convert_to_unicode(encoded_subject) assert decoded == decoded_subject assert 'utf-8' not in decoded def test_unfold(): assert unfold('test\n\tthis') == 'test this' assert unfold('test\r\n\tthis') == 'test this' assert unfold('test \r\n this') == 'test this' def test_email_raw_headers(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multiple_to_cc.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == 'test@test.com' assert results[0]['EntryContext']['Email']['To'] == 'test@test.com, example1@example.com' assert results[0]['EntryContext']['Email']['CC'] == 'test@test.com, example1@example.com' assert results[0]['EntryContext']['Email']['HeadersMap']['From'] == 'Guy Test ' assert results[0]['EntryContext']['Email']['HeadersMap']['To'] == 'Guy Test ' \ ', Guy Test1 ' assert results[0]['EntryContext']['Email']['HeadersMap']['CC'] == 'Guy Test , ' \ 'Guy Test1 ' def test_email_raw_headers_from_is_cyrillic_characters(mocker): """ Given: - The email message the should pe parsed. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multiple_to_cc_from_Cyrillic' '_characters.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == 'no-reply@google.com' assert results[0]['EntryContext']['Email']['To'] == 'test@test.com, example1@example.com' assert results[0]['EntryContext']['Email']['CC'] == 'test@test.com, example1@example.com' assert results[0]['EntryContext']['Email']['HeadersMap']['From'] == u'"✅✅✅ BA ! ' \ u'https://example.com ." ' \ u'' assert results[0]['EntryContext']['Email']['HeadersMap']['To'] == 'Guy Test ' \ ', Guy Test1 ' assert results[0]['EntryContext']['Email']['HeadersMap']['CC'] == 'Guy Test , ' \ 'Guy Test1 ' def test_email_from_one_line_no_comma_lf(mocker): """ Given: - The email message with a 'From' header that contains a newline. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '3'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('from_one_line_no_comma_LF' '.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == '1111@test.org' assert results[0]['EntryContext']['Email']['HeadersMap']['From'] == '\"First Last\" <1111@test.org>' def test_email_from_newline_lf(mocker): """ Given: - The email message with a 'From' header that contains a newline. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '3'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('from_multiple_lines_LF' '.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == '1111@test.org' def test_email_from_newline_crlf(mocker): """ Given: - The email message with a 'From' header that contains a newline. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '3'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('from_multiple_lines_CRLF' '.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == '1111@test.org' def test_email_from_one_line_lf(mocker): """ Given: - The email message with a 'From' header that contains a newline. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '3'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('from_one_line_LF' '.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == '1111@test.org' def test_email_from_one_line_crlf(mocker): """ Given: - The email message with a 'From' header that contains a newline. - Checking an email file that contains '\r\n' in it's 'From' header. When: - After parsed email file into Email object Then: - Validate that all raw headers are valid. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '3'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('from_one_line_CRLF' '.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['From'] == '1111@test.org' def test_eml_contains_eml_with_status(mocker): subject = '=?iso-8859-7?B?Rlc6IEZPT0RMSU5LINDLx9HZzMc=?=' # disable-secrets-detection decoded = convert_to_unicode(subject) subject_attach = decoded.decode('utf-8') mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('ParseEmailFiles-test-emls.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() # assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][1]['Subject'] == subject_attach @pytest.mark.parametrize('email_file', ['eml_contains_base64_eml.eml', 'eml_contains_base64_eml2.eml']) def test_eml_contains_base64_encoded_eml(mocker, email_file): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file(email_file)) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 3 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][0]['Subject'] == 'Fwd: test - inner attachment eml (base64)' assert 'message.eml' in results[0]['EntryContext']['Email'][0]['Attachments'] assert 'message.eml' in results[0]['EntryContext']['Email'][0]['AttachmentsData'][0]['Name'] assert results[0]['EntryContext']['Email'][0]['Depth'] == 0 assert results[0]['EntryContext']['Email'][1]["Subject"] == 'test - inner attachment eml' assert results[0]['EntryContext']['Email'][1]['Depth'] == 1 # check that we parse an email with "data" type and eml extension @pytest.mark.parametrize('file_info', ['data', 'data\n']) def test_eml_data_type(mocker, file_info): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smtp_email_type.eml', info=file_info)) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Test Smtp Email' def test_smime(mocker): multipart_sigened = 'multipart/signed; protocol="application/pkcs7-signature";, ASCII text, with CRLF line terminators' mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smime.p7m', info=multipart_sigened)) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() # assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Testing Email Attachment' def test_smime_msg(mocker): info = 'CDFV2 Microsoft Outlook Message' mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smime-p7s.msg', info=info)) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() # assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'test' def test_msg_headers_map(): email_data, ignore = handle_msg('test_data/utf_subject.msg', 'utf_subject.msg') assert '?utf-8' not in email_data['Subject'] assert 'TESTING' in email_data['Subject'] assert 'This is a test email.' in email_data['Text'] assert 'mobi777@gmail.com' in email_data['From'] assert 47 == len(email_data['HeadersMap']) assert isinstance(email_data['HeadersMap']['Received'], list) assert 8 == len(email_data['HeadersMap']['Received']) assert '1; DM6PR11MB2810; 31:tCNnPn/K8BROQtLwu3Qs1Fz2TjDW+b7RiyfdRvmvCG+dGRQ08+3CN4i8QpLn2o4' \ in email_data['HeadersMap']['X-Microsoft-Exchange-Diagnostics'][2] assert '2eWTrUmQCI=; 20:7yMOvCHfrNUNaJIus4SbwkpcSids8EscckQZzX/oGEwux6FJcH42uCQd9tNH8gmDkvPw' \ in email_data['HeadersMap']['X-Microsoft-Exchange-Diagnostics'][2] assert 'text/plain' in email_data['Format'] def test_parse_body_with_russian_language(): email_data, ignore = handle_msg('test_data/Phishing_TEST.msg', 'Phishing_TEST.msg') assert str(email_data['Text']).startswith('\xd0\xa3') assert str(email_data['HTML']).startswith('\xd0\xa3') def test_unknown_file_type(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smtp_email_type.eml', info="bad")) mocker.patch.object(demisto, 'results') try: main() except SystemExit: gotexception = True assert gotexception results = demisto.results.call_args[0] assert len(results) == 1 assert 'Unknown file format:' in results[0]['Contents'] assert 'smtp_email_type.eml' in results[0]['Contents'] def test_no_content_type_file(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('no_content_type.eml', info="ascii text")) mocker.patch.object(demisto, 'results') main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'No content type' def test_get_msg_mail_format(): msg_mail_format = get_msg_mail_format({ 'Headers': 'Content-type:text/plain;' }) assert msg_mail_format == 'text/plain' msg_mail_format = get_msg_mail_format({ 'Something': 'else' }) assert msg_mail_format == '' msg_mail_format = get_msg_mail_format({ 'Headers': None }) assert msg_mail_format == '' def test_handle_msg_with_attachments(): """ Given: - A msg file with attachments When: - Running the 'handle_msg' method Then: - Ensure that the attachment name is in the results """ result = handle_msg('test_data/html_attachment.msg', 'html_attachment.msg') assert result[0]['Attachments'] == 'dummy-attachment.txt' def test_no_content_file(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('no_content.eml', info="ascii text")) mocker.patch.object(demisto, 'results') try: main() except SystemExit: gotexception = True assert gotexception results = demisto.results.call_args[0] assert len(results) == 1 assert 'Could not extract email from file' in results[0]['Contents'] def test_eml_contains_htm_attachment(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('eml_contains_htm_attachment.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][u'Attachments'] == '1.htm' assert results[0]['EntryContext']['Email'][u'AttachmentsData'][0]['Name'] == '1.htm' def test_signed_attachment(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('email_with_signed_attachment.eml', info="multipart/mixed")) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results[0]['EntryContext']['Email']) == 2 def test_eml_contains_html_and_text(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multipart_alternative_format.p7m', info="multipart/alternative;, " "ISO-8859 text, with CRLF line terminators")) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert "

żółć" in results[0]['EntryContext']['Email']['HTML'] def test_eml_format_multipart_mix(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multipart_mixed_format.p7m', info="multipart/mixed")) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert "Warsaw, Poland " in results[0]['EntryContext']['Email']['HTML'] def test_eml_format_multipart_related(mocker): mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multipart_related_format.p7m', info="multipart/related")) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert "Warsaw, Poland " in results[0]['EntryContext']['Email']['HTML'] def test_eml_base64_header_comment_although_string(mocker): def executeCommand(name, args=None): if name == 'getFilePath': return [ { 'Type': entryTypes['note'], 'Contents': { 'path': 'test_data/DONT_OPEN-MALICIOUS_base64_headers.eml', 'name': 'DONT_OPEN-MALICIOUS_base64_headers.eml' } } ] elif name == 'getEntry': return [ { 'Type': entryTypes['file'], 'FileMetadata': { 'info': 'UTF-8 Unicode text, with very long lines, with CRLF line terminators' } } ] else: raise ValueError('Unimplemented command called: {}'.format(name)) mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=executeCommand) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 3 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][0]['Subject'] == 'DONT OPEN - MALICIOS' assert results[0]['EntryContext']['Email'][0]['Depth'] == 0 assert 'Attacker+email+.msg' in results[0]['EntryContext']['Email'][0]['Attachments'] assert 'Attacker+email+.msg' in results[0]['EntryContext']['Email'][0]['AttachmentsData'][0]['Name'] assert results[0]['EntryContext']['Email'][1]["Subject"] == 'Attacker email' assert results[0]['EntryContext']['Email'][1]['Depth'] == 1 def test_message_rfc822_without_info(mocker): """ Given: - EML file with content type message/rfc822 - Demisto entry metadata returned without info, but with type When: - Running the script on the email file Then: - Verify the script runs successfully - Ensure 2 entries are returned as expected """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': '1'}) mocker.patch.object( demisto, 'executeCommand', side_effect=exec_command_for_file('eml_contains_base64_eml2.eml', info='', file_type='message/rfc822') ) mocker.patch.object(demisto, 'results') main() assert demisto.results.call_count == 2 results = demisto.results.call_args_list assert len(results) == 2 assert results[0][0][0]['Type'] == entryTypes['file'] assert results[1][0][0]['Type'] == entryTypes['note'] assert results[1][0][0]['EntryContext']['Email']['From'] == 'koko@demisto.com' def test_md_output_empty_body_text(): """ Given: - The input email_data where the value of the 'Text' field is None. When: - Running the data_to_md command on this email_data. Then: - Validate that output the md doesn't contain a row for the 'Text' field. """ email_data = { 'To': 'email1@paloaltonetworks.com', 'From': 'email2@paloaltonetworks.com', 'Text': None } expected = u'### Results:\n' \ u'* From:\temail2@paloaltonetworks.com\n' \ u'* To:\temail1@paloaltonetworks.com\n' \ u'* CC:\t\n' \ u'* Subject:\t\n' \ u'* Attachments:\t\n\n\n' \ u'### HeadersMap\n' \ u'**No entries.**\n' md = data_to_md(email_data) assert expected == md email_data = { 'To': 'email1@paloaltonetworks.com', 'From': 'email2@paloaltonetworks.com', } expected = u'### Results:\n' \ u'* From:\temail2@paloaltonetworks.com\n' \ u'* To:\temail1@paloaltonetworks.com\n' \ u'* CC:\t\n' \ u'* Subject:\t\n' \ u'* Attachments:\t\n\n\n' \ u'### HeadersMap\n' \ u'**No entries.**\n' md = data_to_md(email_data) assert expected == md def test_md_output_with_body_text(): """ Given: - The input email_data with a value in the 'Text' field. When: - Running the data_to_md command on this email_data. Then: - Validate that the output md contains a row for the 'Text' field. """ email_data = { 'To': 'email1@paloaltonetworks.com', 'From': 'email2@paloaltonetworks.com', 'Text': '' } expected = u'### Results:\n' \ u'* From:\temail2@paloaltonetworks.com\n' \ u'* To:\temail1@paloaltonetworks.com\n' \ u'* CC:\t\n' \ u'* Subject:\t\n' \ u'* Body/Text:\t[email text]\n' \ u'* Attachments:\t\n\n\n' \ u'### HeadersMap\n' \ u'**No entries.**\n' md = data_to_md(email_data) assert expected == md def test_create_headers_map_empty_headers(): """ Given: - The input headers is None. When: - Running the create_headers_map command on these headers. Then: - Validate that the function does not fail """ msg_dict = { 'From': None, 'CC': None, 'BCC': None, 'To': u'test@demisto.com', 'Depth': 0, 'HeadersMap': {}, 'Attachments': u'image002.png,image003.png,image004.png,image001.png', 'Headers': None, 'Text': u'Hi', 'Subject': u'test' } headers, headers_map = create_headers_map(msg_dict.get('Headers')) assert headers == [] assert headers_map == {} def test_eml_contains_htm_attachment_empty_file(mocker): """ Given: An email containing both an empty text file and a base64 encoded htm file. When: Parsing a valid email file with default parameters. Then: Three entries will be returned to the war room. One containing the command results. Another containing the empty file. The last contains the htm file. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('eml_contains_emptytxt_htm_file.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email'][0]['AttachmentNames'] == ['unknown_file_name0', 'SomeTest.HTM'] def test_eml_contains_attachment_with_unknown_encoded_file_name(mocker): """ Given: An email containing an attachment with unknown encoded name. When: Parsing a valid email file with default parameters. Then: The file name is parsed as expected and the debug alert exist. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('Unknown_encode_attachment_name.eml')) mocker.patch.object(demisto, 'results') mocker.patch.object(demisto, 'debug') main() results = demisto.results.call_args[0] debug = demisto.debug.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['AttachmentNames'] == ['04AIf|???���������������.pdf'] assert debug[0] == 'Could not find the encoding type of the string, decoding by default with utf-8' def test_eml_contains_htm_attachment_empty_file_max_depth(mocker): """ Given: An email containing both an empty text file and a base64 encoded htm file. When: Parsing a valid email file with max_depth=1. Then: One entry containing the command results will be returned to the war room. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test', 'max_depth': 1}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('eml_contains_emptytxt_htm_file.eml')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] def test_double_dots_removed(mocker): """ Fixes: https://github.com/demisto/etc/issues/27229 Given: an eml file with a line break (`=\r\n`) which caused the duplication of dots (`..`). Then: replace the two dots with one and test that `part.get_payload()` decodes it correctly. """ import ParseEmailFiles as pef mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('multiple_to_cc.eml')) mocker.patch.object(pef, 'get_utf_string') main() assert 'http://schemas.microsoft.com/office/2004/12/omml' in pef.get_utf_string.mock_calls[0][1][0] def test_only_parts_of_object_email_saved(mocker): """ Fixes: https://github.com/demisto/etc/issues/29476 Given: an eml file with a line break (`\n`) in the payload that has failed due to wring type. Then: filter only parts that are of type email.message.Message. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('new-line-in-parts.eml')) mocker.patch.object(demisto, 'results') main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['AttachmentNames'] == ['logo5.png', 'logo2.png'] def test_long_subject_and_special_characters(mocker): """ Fixes: https://github.com/demisto/etc/issues/47691 Given: an eml file with a long subject and special characters. Then: assert all the subject is parsed correctly. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('file_with_a_long_subject_and_' 'special_characters.eml')) mocker.patch.object(demisto, 'results') main() results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == u'Those characters : üàéüö will mess with the ' \ u'parsing automation' def test_pkcs7_mime(mocker): """ Given: An email file smime2.p7m of type application/pkcs7-mime and info - MIME entity text, ISO-8859 text, with very long lines, with CRLF line terminators When: Parsing the email. Then: The email is parsed correctly. """ mocker.patch.object(demisto, 'args', return_value={'entryid': 'test'}) mocker.patch.object(demisto, 'executeCommand', side_effect=exec_command_for_file('smime2.p7m', info='MIME entity text, ISO-8859 text, with very long lines,' ' with CRLF line terminators')) mocker.patch.object(demisto, 'results') # validate our mocks are good assert demisto.args()['entryid'] == 'test' main() assert demisto.results.call_count == 1 # call_args is tuple (args list, kwargs). we only need the first one results = demisto.results.call_args[0] assert len(results) == 1 assert results[0]['Type'] == entryTypes['note'] assert results[0]['EntryContext']['Email']['Subject'] == 'Testing signed multipart email' def test_PtypString(): data_value = DataModel.PtypString('IPM.Note') assert data_value == u'IPM.Note' data_value = DataModel.PtypString('I\x00P\x00M\x00.\x00N\x00o\x00t\x00e\x00') assert data_value == u'IPM.Note' data_value = DataModel.PtypString(b'e\x9c\xe6\xb9pe') assert data_value == u'eśćąpe' @pytest.mark.parametrize('payload, answer', [ ('escape', 'escape'), (u'eśćąpe', u'eśćąpe') ]) def test_decode_attachment_payload_non_base64(payload, answer): class MockedMessage: def __init__(self, payload=None): self.payload = payload def get_payload(self): return self.payload from ParseEmailFiles import decode_attachment_payload assert answer == decode_attachment_payload(MockedMessage(payload)) @pytest.mark.parametrize('payload, answer', [ ('//5lAFsBBwEFAXAAZQA=', '\xff\xfee\x00[\x01\x07\x01\x05\x01p\x00e\x00'), # eśćąpe ('ZXNjYXBl', 'escape') ]) def test_decode_attachment_payload_base64(payload, answer): class MockedMessage: def __init__(self, payload=None): self.payload = payload def get_payload(self): return self.payload from ParseEmailFiles import decode_attachment_payload assert answer == decode_attachment_payload(MockedMessage(payload)) @pytest.mark.parametrize('nesting_level_to_return, output, res', [('All files', ['output1'], ('output1', ['output1'])), ('Outer file', ['output1', 'output2', 'output3'], ('output1', 'output1')), ('Inner file', ['output1', 'output2', 'output3'], ('output3', 'output3'))]) def test_parse_nesting_level(nesting_level_to_return, output, res): """ Given: - parsed email output, nesting_level_to_return param - All files. - parsed email output, nesting_level_to_return param - Outer file. - parsed email output, nesting_level_to_return param - Inner file. When: - Getting all nested emails. - Getting only outer email file. - Getting only inner email file. Then: Validate that returned result as expected. """ assert parse_nesting_level(nesting_level_to_return, output) == res