import os import demistomock as demisto import pytest from pytest_mock import MockerFixture from ReadPDFFileV2 import PdfInvalidCredentialsException, PdfPermissionsException CWD = os.getcwd() if os.getcwd().endswith("test_data") else f"{os.getcwd()}/test_data" def open_html_file(file): with open(file, encoding="utf-8") as f: return f.read() def test_extract_hash_contexts(): """ Given - A PDF with hashes in it. When - Trying extract the hashes from the file. Then - Validate that the hashes were extracted successfully. """ from ReadPDFFileV2 import extract_hash_contexts_from_pdf_file, get_pdf_text expected_hash_contexts = [ {"type": "SHA1", "value": "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"}, {"type": "SHA256", "value": "2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824"}, {"type": "SHA256", "value": "8732331accf45f86a00ca823cb24d9806ec1380846a337ac86b4fe6f9d06f1f5"}, {"type": "MD5", "value": "5d41402abc4b2a76b9719d911017c592"}, ] # We first extract the file's text, and then extract the hashes pdf_text_output_path = f"{CWD}/PDFText.txt" file_text = get_pdf_text(f"{CWD}/pdf-with-hashes.pdf", pdf_text_output_path) hash_contexts = extract_hash_contexts_from_pdf_file(file_text) assert len(hash_contexts) == len(expected_hash_contexts) for hash_context in hash_contexts: assert hash_context in expected_hash_contexts def test_hash_contexts_in_return_results(): """ Given - A hash context to add to the entry context. When - Building the entry context. Then - Validate that the hash context was added. """ from ReadPDFFileV2 import build_readpdf_entry_context hashes = { "Hashes": [ {"type": "SHA1", "value": "aaf4c61ddcc5e8a2dabede0f3b482cd9aea9434d"}, {"type": "MD5", "value": "5d41402abc4b2a76b9719d911017c592"}, ] } entry_context = build_readpdf_entry_context(hashes) assert entry_context == hashes UNESCAPE_CASES = [ ( False, { "http://example.com/abc" xmlns:xsi="http://example.com/abc" ", "http://www.w3.org/1999/xhtml", "http://example.com/abc http://example.com/abc/v1.2/1_2.xsd"> ", }, ), (True, {"http://www.w3.org/1999/xhtml", "http://example.com/abc/v1.2/1_2.xsd", "http://example.com/abc"}), ] @pytest.mark.parametrize("unescape_url, urls_set", UNESCAPE_CASES) def test_urls_are_unescaped(unescape_url, urls_set): """ Given - A pdf file that has xml content in it. - Whether to unescape the html content or no. When - Trying extract the urls from an html with escaping characters. Then - The set of urls are extracted correctly, with respect to whether to unescape them or not. """ from ReadPDFFileV2 import get_urls_and_emails_from_pdf_html_content urls, _ = get_urls_and_emails_from_pdf_html_content(f"{CWD}/xml_with_urls.pdf", CWD, unescape_url) assert urls == urls_set def test_urls_are_found_correctly(mocker): """ Given - a pdf html content. When - trying extract the urls from that html. Then - the correct url is extracted from the html content. """ from ReadPDFFileV2 import get_urls_and_emails_from_pdf_html_content mocker.patch("ReadPDFFileV2.get_pdf_htmls_content", return_value=open_html_file(f"{CWD}/pdf-html-content.html")) urls, _ = get_urls_and_emails_from_pdf_html_content("", "") assert urls == {"http://www.w3.org/1999/xhtml"} def test_run_shell_command_using_owner_password_error(): """ Given - An encrypted pdf file (where the user and owner password are not the same) and an owner password. When - Trying to decrypt the file with the owner password using the [upw] flag (which specifies a user password). Then - Raise an exception since we need to use the [opw] flag (which specifies an owner password). """ from ReadPDFFileV2 import run_shell_command with pytest.raises(PdfInvalidCredentialsException) as e: run_shell_command("pdfinfo", "-upw", "123456!", f"{CWD}/dummy-with-owner-pass.pdf") assert "Incorrect password" in str(e) def test_run_shell_command_using_owner_password(): """ Given - An encrypted pdf file (where the user and owner password are not the same) and an owner password. When - Decrypting the file with the owner password using the [opw] flag (which specifies an owner password). Then - Validate that the function did not raise any errors. """ from ReadPDFFileV2 import run_shell_command run_shell_command("pdfinfo", "-opw", "123456!", f"{CWD}/dummy-with-owner-pass.pdf") def test_get_pdf_metadata_using_owner_password(mocker: MockerFixture): """ Given - An encrypted pdf file (where the user and owner password are not the same) and an owner password. When - Extracting the metadata of the file. Then - Validate that pdfinfo was first called using the [upw] flag, and then the [opw] flag. """ from ReadPDFFileV2 import get_pdf_metadata, run_shell_command run_shell_command_mocker = mocker.patch("ReadPDFFileV2.run_shell_command", side_effect=run_shell_command) get_pdf_metadata(file_path=f"{CWD}/dummy-with-owner-pass.pdf", user_or_owner_password="123456!") assert run_shell_command_mocker.call_count == 2 assert run_shell_command_mocker.call_args_list[0][0][0:2] == ("pdfinfo", "-upw") assert run_shell_command_mocker.call_args_list[1][0][0:2] == ("pdfinfo", "-opw") def test_incorrect_authentication(): """ Given - An encrypted pdf file and an incorrect password. When - Trying to decrypt the file(using the password) to extract data. Then - The program will catch this error and raise the appropriate exception. """ from ReadPDFFileV2 import get_pdf_metadata, handling_pdf_credentials file_path = f"{CWD}/encrypted.pdf" dec_file_path = f"{CWD}/decrypted.pdf" with pytest.raises(PdfInvalidCredentialsException) as e: get_pdf_metadata(file_path=file_path, user_or_owner_password="12") assert "Incorrect password" in str(e) with pytest.raises(PdfInvalidCredentialsException) as e: handling_pdf_credentials(cpy_file_path=file_path, dec_file_path=dec_file_path, encrypted="yes", user_password="12") assert "Incorrect password" in str(e) def test_get_files_names_in_path(): from ReadPDFFileV2 import get_files_names_in_path pdf_file_names = get_files_names_in_path("test_data", "*.pdf") assert "scanned.pdf" in pdf_file_names pdf_file_names = get_files_names_in_path("test_data", "*.pdf", full_path=True) assert "test_data/text-only.pdf" in pdf_file_names def test_get_images_paths_in_path(): from ReadPDFFileV2 import get_images_paths_in_path img_file_paths = get_images_paths_in_path("test_data") assert "test_data/test1.png" in img_file_paths assert "test_data/scanned.pdf" not in img_file_paths ENC_PDF_META_DATA_CASES = [ ( ( b"Title: sample1.pdf\nKeywords: \nCreator: Preview\n" b"Producer: macOS Version 10.14.4 (Build 18E226) Quartz PDFContext\nCreationDate: Wed May 15 08:18:48 2019\n" b"ModDate: Wed May 15 08:18:48 2019\nTagged: no\nForm: none\nPages: 2\n" b"Encrypted: AES 128-bit\nPermissions: print:yes copy:yes change:yes addNotes:yes\n" b"Page size: 595 x 842 pts (A4) (rotated 0 degrees)\nFile size: 71085 bytes\nOptimized: no\n" b"PDF version: 1.6\n" ), { "Title": "sample1.pdf", "Keywords": "", "Creator": "Preview", "Producer": "macOS Version 10.14.4 (Build 18E226) Quartz PDFContext", "CreationDate": "Wed May 15 08:18:48 2019", "ModDate": "Wed May 15 08:18:48 2019", "Tagged": "no", "Form": "none", "Pages": "2", "Encrypted": "AES 128-bit", "Permissions": "print:yes copy:yes change:yes addNotes:yes", "PageSize": "595 x 842 pts (A4) (rotated 0 degrees)", "FileSize": "71085 bytes", "Optimized": "no", "PDFVersion": "1.6", }, ) ] @pytest.mark.parametrize("raw_result, expected_result", ENC_PDF_META_DATA_CASES) def test_get_pdf_metadata_with_encrypted(mocker, raw_result, expected_result): from ReadPDFFileV2 import get_pdf_metadata file_path = f"{CWD}/encrypted.pdf" mocker.patch("ReadPDFFileV2.run_shell_command", return_value=raw_result) metadata = get_pdf_metadata(file_path, user_or_owner_password="1234") assert metadata == expected_result PDF_META_DATA_CASES = [ ( ( b"Title: Microsoft Word - Document1\nKeywords: \nCreator: Word\n" b"Producer: macOS Version 10.14.4 (Build 18E226) Quartz PDFContext\nCreationDate: Wed May 15 11:47:28 2019\n" b"ModDate: Wed May 15 11:47:28 2019\nTagged: no\nForm: none\nPages: 1\n" b"Encrypted: no\nPage size: 595 x 842 pts (A4) (rotated 0 degrees)\nFile size: 18920 bytes\n" b"Optimized: no\nPDF version: 1.3\n" ), { "Title": "Microsoft Word - Document1", "Keywords": "", "Creator": "Word", "Producer": "macOS Version 10.14.4 (Build 18E226) Quartz PDFContext", "CreationDate": "Wed May 15 11:47:28 2019", "ModDate": "Wed May 15 11:47:28 2019", "Tagged": "no", "Form": "none", "Pages": "1", "Encrypted": "no", "PageSize": "595 x 842 pts (A4) (rotated 0 degrees)", "FileSize": "18920 bytes", "Optimized": "no", "PDFVersion": "1.3", }, ) ] @pytest.mark.parametrize("raw_result, expected_result", PDF_META_DATA_CASES) def test_get_metadata_without_encrypted(mocker, raw_result, expected_result): from ReadPDFFileV2 import get_pdf_metadata try: get_pdf_metadata(f"{CWD}/encrypted.pdf") raise Exception("Incorrect password exception should've been thrown") except PdfPermissionsException as e: assert "Incorrect password" in str(e) mocker.patch("ReadPDFFileV2.run_shell_command", return_value=raw_result) metadata = get_pdf_metadata(f"{CWD}/text-only.pdf") assert metadata == expected_result def test_get_pdf_text_with_encrypted(tmp_path): from ReadPDFFileV2 import get_pdf_text, handling_pdf_credentials file_path = f"{CWD}/encrypted.pdf" dec_file_path = f"{CWD}/decrypted.pdf" dec_file_path = handling_pdf_credentials( cpy_file_path=file_path, user_password="1234", dec_file_path=dec_file_path, encrypted="yes" ) text = get_pdf_text(dec_file_path, f"{tmp_path}/encrypted.txt") expected = ( "XSL FO Sample Copyright © 2002-2005 Antenna House, Inc. All rights reserved.\n\n" "Links in PDF\nPDF link is classified into two parts, link to the specified position in the PDF " "document, and link to the external document.\n" "The internal-destination property of fo:basic-link indicates to link to the position in the same" " document. The externaldestination property indicates to link to external document. " "Below shows the example.\n\nExample of a link to internal destination\nRefer to Purchasing " "Assistance to get more information.\nExample of a link to external destination\nRefer to Purchasing " "Assistance to get more information." ) if os.path.exists(dec_file_path): os.remove(dec_file_path) assert text.startswith(expected) def test_get_pdf_text_without_encrypted(tmp_path): from ReadPDFFileV2 import get_pdf_text # assert error raised try: get_pdf_text(f"{CWD}/encrypted.pdf", f"{tmp_path}/encrypted.txt") raise Exception("Incorrect password exception should've been thrown") except PdfInvalidCredentialsException as e: assert "Incorrect password" in str(e) # assert not warnings are raised text = get_pdf_text(f"{CWD}/warning_trigger.pdf", f"{tmp_path}/warning_trigger.txt") assert "Riu Plaza Berlin" in text # assert extract file correctly text = get_pdf_text(f"{CWD}/text-only.pdf", f"{tmp_path}/text-only.txt") expected = "עברית" assert expected in text assert text.startswith("This is a pdf document with a text line within it.") text = get_pdf_text(f"{CWD}/text-with-images.pdf", f"{tmp_path}/text-with-images.txt") expected = "Create an ETD Using Adobe Acrobat" assert text.startswith(expected) text = get_pdf_text(f"{CWD}/scanned.pdf", f"{tmp_path}/scanned.txt") expected = "\x0c" assert expected == text def test_get_pdf_htmls_content_with_encrypted(mocker, tmp_path): mocker.patch.object(demisto, "args", return_value={"userPassword": "1234"}) from ReadPDFFileV2 import get_images_paths_in_path, get_pdf_htmls_content, handling_pdf_credentials file_path = f"{CWD}/encrypted.pdf" dec_file_path = f"{CWD}/decrypted.pdf" dec_file_path = handling_pdf_credentials( cpy_file_path=file_path, user_password="1234", dec_file_path=dec_file_path, encrypted="yes" ) # to_html_output_folder = f'{tmp_path}/PDF_html' html_text = get_pdf_htmls_content(dec_file_path, tmp_path) expected = ( "If you are end user who wishes to use XSL Formatter yourself, you may purchase " "from our Reseller or direct from Antenna
House.
" ) if os.path.exists(dec_file_path): os.remove(dec_file_path) assert len(get_images_paths_in_path(tmp_path)) != 0, "Failed to get images from html" assert expected in html_text def test_get_pdf_htmls_content_without_encrypted(tmp_path): from ReadPDFFileV2 import get_images_paths_in_path, get_pdf_htmls_content try: get_pdf_htmls_content(f"{CWD}/encrypted.pdf", tmp_path) raise Exception("Incorrect password exception should've been thrown") except PdfPermissionsException as e: assert "Incorrect password" in str(e) # to_html_output_folder = f'{tmp_path}/PDF_html' html_text = get_pdf_htmls_content(f"{CWD}/hyperlinks.pdf", tmp_path) assert "http://www.example.com/" in html_text assert len(get_images_paths_in_path(tmp_path)) != 0, "Failed to get images from html" def test_get_urls_from_binary_file(): from ReadPDFFileV2 import get_urls_from_binary_file urls = get_urls_from_binary_file(f"{CWD}/text-with-images.pdf") assert len(urls) == 10 def test_build_readpdf_entry_object_empty_extract(mocker): from ReadPDFFileV2 import DEFAULT_NUM_IMAGES, build_readpdf_entry_object mocker.patch.object(demisto, "executeCommand", return_value=[{"Contents": ""}]) res = build_readpdf_entry_object("test", {}, "", [], [], [], DEFAULT_NUM_IMAGES) assert res[0]["HumanReadable"] == "### Metadata\n\n### URLs\n\n### Text\n" @pytest.mark.parametrize( "file_path", [ "URLs_Extraction_Test_PDF_Encoding_Google_Docs_Renderer_protected.pdf", "URLs_Extraction_Test_PDF_Encoding_Quartz_PDFContext_protected.pdf", ], ) def test_get_urls_and_emails_from_pdf_annots_with_encrypt(file_path): """ This test verifies URL and Emails extraction from an encrypted PDF file. Given: A path to an encrypted PDF file with a certain encoding: 1. A pdf created with google docs. 2. A pdf created with mac os Notes. Both PDFs include URLs and Email addresses from different kinds that should be extracted: * 'https://test1.com/' - A text url ended with a slash /. * 'https://test2.com' - A text url ended without a slash /. * 'www.test3.net' - A text url without the http prefix. * 'user@test4.com' - A text email address. * 'https://test5.com.co/ed/trn/update?email=user@test6.net' - A text url with an https prefix, and an email address in it. * 'http://www.test7.com' - A text hyperlink of a url. * 'https://test8.com/' - An embedded url (a url that is hyperlinked to an image). When: Running 'get_urls_and_emails_from_pdf_annots' function on the PDF file. Then: Verify that the URLs Emails was extracted successfully. """ from ReadPDFFileV2 import get_urls_and_emails_from_pdf_annots, handling_pdf_credentials expected_urls = { "https://test1.com", "https://test2.com", "http://www.test3.net", "https://test5.com.co/ed/trn/update?email=user@test6.net", "http://www.test7.com", "https://test8.com", } expected_emails = {"user@test4.com", "user@test6.net"} # Decrypt the PDF: dec_file_path = f"{CWD}/decrypted.pdf" file_path = f"{CWD}/{file_path}" dec_file_path = handling_pdf_credentials( cpy_file_path=file_path, user_password="123456", dec_file_path=dec_file_path, encrypted="" ) # decrypt_pdf_file(file_path, '1234', dec_file_path) # Extract URLs and Emails: urls, emails = get_urls_and_emails_from_pdf_annots(dec_file_path) # Delete Decrypted file: if os.path.exists(dec_file_path): os.remove(dec_file_path) assert urls == expected_urls assert emails == expected_emails @pytest.mark.parametrize( "file_path", ["URLs_Extraction_Test_PDF_Encoding_Google_Docs_Renderer.pdf", "URLs_Extraction_Test_PDF_Encoding_Quartz_PDFContext.pdf"], ) def test_get_urls_and_emails_from_pdf_annots_without_encrypt(file_path): """ This test verifies URL and Emails extraction from a non-encrypted PDF file. Given: A path to a PDF file with a certain encoding: 1. A pdf created with google docs. 2. A pdf created with mac os Notes. Both PDFs include URLs and Email addresses from different kinds that should be extracted: * 'https://test1.com/' - A text url ended with a slash /. * 'https://test2.com' - A text url ended without a slash /. * 'www.test3.net' - A text url without the http prefix. * 'user@test4.com' - A text email address. * 'https://test5.com.co/ed/trn/update?email=user@test6.net' - A text url with an https prefix, and an email address in it. * 'http://www.test7.com' - A text hyperlink of a url. * 'https://test8.com/' - An embedded url (a url that is hyperlinked to an image). When: Running 'get_urls_and_emails_from_pdf_annots' function on the PDF file. Then: Verify that the URLs Emails was extracted successfully. """ from ReadPDFFileV2 import get_urls_and_emails_from_pdf_annots expected_urls = { "https://test1.com", "https://test2.com", "http://www.test3.net", "https://test5.com.co/ed/trn/update?email=user@test6.net", "http://www.test7.com", "https://test8.com", } expected_emails = {"user@test4.com", "user@test6.net"} file_path = f"{CWD}/{file_path}" # Extract URLs and Emails: urls, emails = get_urls_and_emails_from_pdf_annots(file_path) assert urls == expected_urls assert emails == expected_emails def test_get_urls_and_emails_from_pdf_file_with_encrypt(tmp_path): """ This test verifies URL and Emails extraction from an encrypted PDF file. Given: A path to an encrypted PDF file with a certain encoding (Libreoffice Encoding). When: Running 'extract_urls_and_emails_from_pdf_file' function on the PDF file. Then: Verify that the URLs Emails was extracted successfully. """ from ReadPDFFileV2 import extract_urls_and_emails_from_pdf_file, handling_pdf_credentials expected_urls = { "www.hiddenvirusaddress.cn", "www.msn.com", "http://www.docxtesturl.com", "www.google.com", "www.docxtesturl.com", "http://www.msn.com", } expected_emails = { "Userthatdoesnotexist3@demis", "userthatdoesnotexist@demisto.com", "userthatdoesnotexist4@demis", "Userthatdoesnotexist2@demisto.com", } # Decrypt the PDF: file_path = f"{CWD}/URLs_Extraction_Test_PDF_Encoding_LibreOffice_protected.pdf" dec_file_path = f"{CWD}/decrypted.pdf" dec_file_path = handling_pdf_credentials( cpy_file_path=file_path, user_password="123456", dec_file_path=dec_file_path, encrypted="" ) # decrypt_pdf_file(file_path, '123456', dec_file_path) # Extract URLs and Emails: urls, emails = extract_urls_and_emails_from_pdf_file(dec_file_path, tmp_path) # Delete Decrypted file: if os.path.exists(dec_file_path): os.remove(dec_file_path) assert {url_data["Data"] for url_data in urls} == expected_urls assert set(emails) == expected_emails def test_get_urls_and_emails_from_pdf_file_without_encrypt(tmp_path): """ This test verifies URL and Emails extraction from a non-encrypted PDF file. Given: A path to a PDF file with a certain encoding (Libreoffice Encoding). When: Running 'extract_urls_and_emails_from_pdf_file' function on the PDF file. Then: Verify that the URLs Emails was extracted successfully. """ from ReadPDFFileV2 import extract_urls_and_emails_from_pdf_file expected_urls = { "www.hiddenvirusaddress.cn", "www.msn.com", "http://www.docxtesturl.com", "www.google.com", "www.docxtesturl.com", "http://www.msn.com", } expected_emails = { "Userthatdoesnotexist3@demis", "userthatdoesnotexist@demisto.com", "userthatdoesnotexist4@demis", "Userthatdoesnotexist2@demisto.com", } file_path = f"{CWD}/URLs_Extraction_Test_PDF_Encoding_LibreOffice.pdf" # Extract URLs and Emails: urls, emails = extract_urls_and_emails_from_pdf_file(file_path, tmp_path) assert set(emails) == expected_emails assert {url_data["Data"] for url_data in urls} == expected_urls def test_handle_error_read_only(mocker): from ReadPDFFileV2 import handle_error_read_only mocker.patch("ReadPDFFileV2.os.access", return_value=False) def fun(path): return path change_permition = mocker.patch("ReadPDFFileV2.os.chmod") handle_error_read_only( fun, f"{CWD}/test_for_read_only_file.txt", "The error is not due to a problem with write permissions to the file" ) assert change_permition.call_count == 1 def test_handle_error_read_only_failed(mocker): from ReadPDFFileV2 import handle_error_read_only mocker.patch("ReadPDFFileV2.os.access", return_value=True) def fun(path): return path with pytest.raises(Exception) as e: handle_error_read_only( fun, f"{CWD}/test_for_read_only_file.txt", "The error is not due to a problem with write permissions to the file" ) assert str(e.value) == "The error is not due to a problem with write permissions to the file" def test_extract_urls_and_emails_from_annot_objects_with_binary_data(mocker): """ Given: A list of annotation objects where one object contains binary data. When: The extract_urls_and_emails_from_annot_objects function is called with these objects. Then: The function should correctly decode the binary data and extract the URL and email. """ from ReadPDFFileV2 import extract_urls_and_emails_from_annot_objects mock_annot_object = mocker.Mock() mock_annot_object.get_object.return_value = mocker.Mock() binary_data = b"https://example.com user@example.com" mocker.patch("ReadPDFFileV2.extract_url_from_annot_object", return_value=binary_data) urls, emails = extract_urls_and_emails_from_annot_objects([mock_annot_object]) assert urls == {"https://example.com"} assert emails == {"user@example.com"}