A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Chore: use pathlib in remaining tests
shamoonbc2facc87f61
diff --git a/pyproject.toml b/pyproject.tomlindex 7e8ad1f61..a32a672b2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -221,15 +221,6 @@ lint.per-file-ignores."src/documents/parsers.py" = [lint.per-file-ignores."src/documents/signals/handlers.py" = ["PTH",] # TODO Enable & remove-lint.per-file-ignores."src/documents/tests/test_consumer.py" = [- "PTH",-] # TODO Enable & remove-lint.per-file-ignores."src/documents/tests/test_file_handling.py" = [- "PTH",-] # TODO Enable & remove-lint.per-file-ignores."src/documents/tests/test_migration_archive_files.py" = [- "PTH",-] # TODO Enable & removelint.per-file-ignores."src/documents/views.py" = ["PTH",] # TODO Enable & removediff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.pyindex 3c17ddfaf..8d1b11ec1 100644--- a/src/documents/tests/test_consumer.py+++ b/src/documents/tests/test_consumer.py@@ -1,5 +1,4 @@import datetime-import osimport shutilimport statimport tempfile@@ -66,7 +65,7 @@ class CopyParser(_BaseTestParser):def parse(self, document_path, mime_type, file_name=None):self.text = "The text"- self.archive_path = os.path.join(self.tempdir, "archive.pdf")+ self.archive_path = Path(self.tempdir / "archive.pdf")shutil.copy(document_path, self.archive_path)@@ -96,15 +95,16 @@ class FaultyGenericExceptionParser(_BaseTestParser):def fake_magic_from_file(file, *, mime=False):if mime:- if file.name.startswith("invalid_pdf"):+ filepath = Path(file)+ if filepath.name.startswith("invalid_pdf"):return "application/octet-stream"- if os.path.splitext(file)[1] == ".pdf":+ if filepath.suffix == ".pdf":return "application/pdf"- elif os.path.splitext(file)[1] == ".png":+ elif filepath.suffix == ".png":return "image/png"- elif os.path.splitext(file)[1] == ".webp":+ elif filepath.suffix == ".webp":return "image/webp"- elif os.path.splitext(file)[1] == ".eml":+ elif filepath.suffix == ".eml":return "message/rfc822"else:return "unknown"@@ -225,7 +225,7 @@ class TestConsumer(self.assertEqual(document.content, "The Text")self.assertEqual(document.title,- os.path.splitext(os.path.basename(filename))[0],+ Path(filename).stem,)self.assertIsNone(document.correspondent)self.assertIsNone(document.document_type)@@ -254,7 +254,7 @@ class TestConsumer(# https://github.com/jonaswinkler/paperless-ng/discussions/1037filename = self.get_test_file()- shadow_file = os.path.join(self.dirs.scratch_dir, "._sample.pdf")+ shadow_file = Path(self.dirs.scratch_dir / "._sample.pdf")shutil.copy(filename, shadow_file)@@ -1082,8 +1082,8 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):outfile.write("echo This message goes to stderr >&2")# Make the file executable- st = os.stat(script.name)- os.chmod(script.name, st.st_mode | stat.S_IEXEC)+ st = Path(script.name).stat()+ Path(script.name).chmod(st.st_mode | stat.S_IEXEC)with override_settings(PRE_CONSUME_SCRIPT=script.name):with self.assertLogs("paperless.consumer", level="INFO") as cm:@@ -1114,8 +1114,8 @@ class PreConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):outfile.write("exit 100\n")# Make the file executable- st = os.stat(script.name)- os.chmod(script.name, st.st_mode | stat.S_IEXEC)+ st = Path(script.name).stat()+ Path(script.name).chmod(st.st_mode | stat.S_IEXEC)with override_settings(PRE_CONSUME_SCRIPT=script.name):with self.get_consumer(self.test_file) as c:@@ -1237,8 +1237,8 @@ class PostConsumeTestCase(DirectoriesMixin, GetConsumerMixin, TestCase):outfile.write("exit -500\n")# Make the file executable- st = os.stat(script.name)- os.chmod(script.name, st.st_mode | stat.S_IEXEC)+ st = Path(script.name).stat()+ Path(script.name).chmod(st.st_mode | stat.S_IEXEC)with override_settings(POST_CONSUME_SCRIPT=script.name):doc = Document.objects.create(title="Test", mime_type="application/pdf")diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.pyindex fe033a267..a49bd07aa 100644--- a/src/documents/tests/test_file_handling.py+++ b/src/documents/tests/test_file_handling.py@@ -1,6 +1,5 @@import datetimeimport logging-import osimport tempfilefrom pathlib import Pathfrom unittest import mock@@ -71,7 +70,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):# test that creating dirs for the source_path creates the correct directorycreate_source_path_directory(document.source_path)Path(document.source_path).touch()- self.assertIsDir(os.path.join(settings.ORIGINALS_DIR, "none"))+ self.assertIsDir(settings.ORIGINALS_DIR / "none")# Set a correspondent and save the documentdocument.correspondent = Correspondent.objects.get_or_create(name="test")[0]@@ -108,7 +107,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):)# Make the folder read- and execute-only (no writing and no renaming)- os.chmod(os.path.join(settings.ORIGINALS_DIR, "none"), 0o555)+ (settings.ORIGINALS_DIR / "none").chmod(0o555)# Set a correspondent and save the documentdocument.correspondent = Correspondent.objects.get_or_create(name="test")[0]@@ -120,7 +119,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):)self.assertEqual(document.filename, "none/none.pdf")- os.chmod(os.path.join(settings.ORIGINALS_DIR, "none"), 0o777)+ (settings.ORIGINALS_DIR / "none").chmod(0o777)@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")def test_file_renaming_database_error(self):@@ -160,7 +159,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):# Check proper handling of filesself.assertIsFile(document.source_path)self.assertIsFile(- os.path.join(settings.ORIGINALS_DIR, "none/none.pdf"),+ settings.ORIGINALS_DIR / "none" / "none.pdf",)self.assertEqual(document.filename, "none/none.pdf")@@ -183,9 +182,9 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):document.delete()empty_trash([document.pk])self.assertIsNotFile(- os.path.join(settings.ORIGINALS_DIR, "none", "none.pdf"),+ settings.ORIGINALS_DIR / "none" / "none.pdf",)- self.assertIsNotDir(os.path.join(settings.ORIGINALS_DIR, "none"))+ self.assertIsNotDir(settings.ORIGINALS_DIR / "none")@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}",@@ -206,15 +205,15 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):Path(document.source_path).touch()# Ensure file was moved to trash after delete- self.assertIsNotFile(os.path.join(settings.EMPTY_TRASH_DIR, "none", "none.pdf"))+ self.assertIsNotFile(Path(settings.EMPTY_TRASH_DIR) / "none" / "none.pdf")document.delete()empty_trash([document.pk])self.assertIsNotFile(- os.path.join(settings.ORIGINALS_DIR, "none", "none.pdf"),+ settings.ORIGINALS_DIR / "none" / "none.pdf",)- self.assertIsNotDir(os.path.join(settings.ORIGINALS_DIR, "none"))- self.assertIsFile(os.path.join(settings.EMPTY_TRASH_DIR, "none.pdf"))- self.assertIsNotFile(os.path.join(settings.EMPTY_TRASH_DIR, "none_01.pdf"))+ self.assertIsNotDir(settings.ORIGINALS_DIR / "none")+ self.assertIsFile(Path(settings.EMPTY_TRASH_DIR) / "none.pdf")+ self.assertIsNotFile(Path(settings.EMPTY_TRASH_DIR) / "none_01.pdf")# Create an identical document and ensure it is trashed under a new namedocument = Document()@@ -227,7 +226,7 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):Path(document.source_path).touch()document.delete()empty_trash([document.pk])- self.assertIsFile(os.path.join(settings.EMPTY_TRASH_DIR, "none_01.pdf"))+ self.assertIsFile(Path(settings.EMPTY_TRASH_DIR) / "none_01.pdf")@override_settings(FILENAME_FORMAT="{correspondent}/{correspondent}")def test_document_delete_nofile(self):@@ -261,8 +260,8 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):document.save()# Check proper handling of files- self.assertIsDir(os.path.join(settings.ORIGINALS_DIR, "test"))- self.assertIsDir(os.path.join(settings.ORIGINALS_DIR, "none"))+ self.assertIsDir(settings.ORIGINALS_DIR / "test")+ self.assertIsDir(settings.ORIGINALS_DIR / "none")self.assertIsFile(important_file)@override_settings(FILENAME_FORMAT="{document_type} - {title}")@@ -371,16 +370,16 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):Path(document.source_path).touch()# Check proper handling of files- self.assertIsDir(os.path.join(settings.ORIGINALS_DIR, "none/none"))+ self.assertIsDir(settings.ORIGINALS_DIR / "none" / "none")document.delete()empty_trash([document.pk])self.assertIsNotFile(- os.path.join(settings.ORIGINALS_DIR, "none/none/none.pdf"),+ settings.ORIGINALS_DIR / "none" / "none" / "none.pdf",)- self.assertIsNotDir(os.path.join(settings.ORIGINALS_DIR, "none/none"))- self.assertIsNotDir(os.path.join(settings.ORIGINALS_DIR, "none"))+ self.assertIsNotDir(settings.ORIGINALS_DIR / "none" / "none")+ self.assertIsNotDir(settings.ORIGINALS_DIR / "none")self.assertIsDir(settings.ORIGINALS_DIR)@override_settings(FILENAME_FORMAT="{doc_pk}")@@ -415,12 +414,12 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):(tmp / "notempty" / "empty").mkdir(exist_ok=True, parents=True)delete_empty_directories(- os.path.join(tmp, "notempty", "empty"),+ tmp / "notempty" / "empty",root=settings.ORIGINALS_DIR,)- self.assertIsDir(os.path.join(tmp, "notempty"))- self.assertIsFile(os.path.join(tmp, "notempty", "file"))- self.assertIsNotDir(os.path.join(tmp, "notempty", "empty"))+ self.assertIsDir(tmp / "notempty")+ self.assertIsFile(tmp / "notempty" / "file")+ self.assertIsNotDir(tmp / "notempty" / "empty")@override_settings(FILENAME_FORMAT="{% if x is None %}/{title]")def test_invalid_format(self):@@ -585,8 +584,8 @@ class TestFileHandling(DirectoriesMixin, FileSystemAssertsMixin, TestCase):class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, TestCase):@override_settings(FILENAME_FORMAT=None)def test_create_no_format(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -604,8 +603,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{correspondent}/{title}")def test_create_with_format(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -632,8 +631,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{correspondent}/{title}")def test_move_archive_gone(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()doc = Document.objects.create(mime_type="application/pdf",@@ -651,9 +650,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{correspondent}/{title}")def test_move_archive_exists(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")- existing_archive_file = os.path.join(settings.ARCHIVE_DIR, "none", "my_doc.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"+ existing_archive_file = settings.ARCHIVE_DIR / "none" / "my_doc.pdf"Path(original).touch()Path(archive).touch()(settings.ARCHIVE_DIR / "none").mkdir(parents=True, exist_ok=True)@@ -676,8 +675,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{title}")def test_move_original_only(self):- original = os.path.join(settings.ORIGINALS_DIR, "document_01.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "document.pdf")+ original = settings.ORIGINALS_DIR / "document_01.pdf"+ archive = settings.ARCHIVE_DIR / "document.pdf"Path(original).touch()Path(archive).touch()@@ -698,8 +697,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{title}")def test_move_archive_only(self):- original = os.path.join(settings.ORIGINALS_DIR, "document.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "document_01.pdf")+ original = settings.ORIGINALS_DIR / "document.pdf"+ archive = settings.ARCHIVE_DIR / "document_01.pdf"Path(original).touch()Path(archive).touch()@@ -725,13 +724,13 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Testif "archive" in str(src):raise OSErrorelse:- os.remove(src)+ Path(src).unlink()Path(dst).touch()m.side_effect = fake_rename- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -751,8 +750,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{correspondent}/{title}")def test_move_file_gone(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"# Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -776,13 +775,13 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Testif "original" in str(src):raise OSErrorelse:- os.remove(src)+ Path(src).unlink()Path(dst).touch()m.side_effect = fake_rename- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -802,8 +801,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="")def test_archive_deleted(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document.objects.create(@@ -830,9 +829,9 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{title}")def test_archive_deleted2(self):- original = os.path.join(settings.ORIGINALS_DIR, "document.webp")- original2 = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "document.webp"+ original2 = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(original2).touch()Path(archive).touch()@@ -865,8 +864,8 @@ class TestFileHandlingWithArchive(DirectoriesMixin, FileSystemAssertsMixin, Test@override_settings(FILENAME_FORMAT="{correspondent}/{title}")def test_database_error(self):- original = os.path.join(settings.ORIGINALS_DIR, "0000001.pdf")- archive = os.path.join(settings.ARCHIVE_DIR, "0000001.pdf")+ original = settings.ORIGINALS_DIR / "0000001.pdf"+ archive = settings.ARCHIVE_DIR / "0000001.pdf"Path(original).touch()Path(archive).touch()doc = Document(diff --git a/src/documents/tests/test_migration_archive_files.py b/src/documents/tests/test_migration_archive_files.pyindex 5039e13de..e5ad44b9e 100644--- a/src/documents/tests/test_migration_archive_files.py+++ b/src/documents/tests/test_migration_archive_files.py@@ -1,6 +1,5 @@import hashlibimport importlib-import osimport shutilfrom pathlib import Pathfrom unittest import mock@@ -21,7 +20,7 @@ migration_1012_obj = importlib.import_module(def archive_name_from_filename(filename):- return os.path.splitext(filename)[0] + ".pdf"+ return Path(filename).stem + ".pdf"def archive_path_old(self):@@ -30,12 +29,12 @@ def archive_path_old(self):else:fname = f"{self.pk:07}.pdf"- return os.path.join(settings.ARCHIVE_DIR, fname)+ return Path(settings.ARCHIVE_DIR) / fnamedef archive_path_new(doc):if doc.archive_filename is not None:- return os.path.join(settings.ARCHIVE_DIR, str(doc.archive_filename))+ return Path(settings.ARCHIVE_DIR) / str(doc.archive_filename)else:return None@@ -48,7 +47,7 @@ def source_path(doc):if doc.storage_type == STORAGE_TYPE_GPG:fname += ".gpg" # pragma: no cover- return os.path.join(settings.ORIGINALS_DIR, fname)+ return Path(settings.ORIGINALS_DIR) / fnamedef thumbnail_path(doc):@@ -56,7 +55,7 @@ def thumbnail_path(doc):if doc.storage_type == STORAGE_TYPE_GPG:file_name += ".gpg"- return os.path.join(settings.THUMBNAIL_DIR, file_name)+ return Path(settings.THUMBNAIL_DIR) / file_namedef make_test_document(@@ -76,7 +75,7 @@ def make_test_document(doc.save()shutil.copy2(original, source_path(doc))- with open(original, "rb") as f:+ with Path(original).open("rb") as f:doc.checksum = hashlib.md5(f.read()).hexdigest()if archive:@@ -86,7 +85,7 @@ def make_test_document(else:shutil.copy2(archive, archive_path_old(doc))- with open(archive, "rb") as f:+ with Path(archive).open("rb") as f:doc.archive_checksum = hashlib.md5(f.read()).hexdigest()doc.save()@@ -96,25 +95,17 @@ def make_test_document(return doc-simple_jpg = os.path.join(os.path.dirname(__file__), "samples", "simple.jpg")-simple_pdf = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")-simple_pdf2 = os.path.join(- os.path.dirname(__file__),- "samples",- "documents",- "originals",- "0000002.pdf",+simple_jpg = Path(__file__).parent / "samples" / "simple.jpg"+simple_pdf = Path(__file__).parent / "samples" / "simple.pdf"+simple_pdf2 = (+ Path(__file__).parent / "samples" / "documents" / "originals" / "0000002.pdf")-simple_pdf3 = os.path.join(- os.path.dirname(__file__),- "samples",- "documents",- "originals",- "0000003.pdf",+simple_pdf3 = (+ Path(__file__).parent / "samples" / "documents" / "originals" / "0000003.pdf")-simple_txt = os.path.join(os.path.dirname(__file__), "samples", "simple.txt")-simple_png = os.path.join(os.path.dirname(__file__), "samples", "simple-noalpha.png")-simple_png2 = os.path.join(os.path.dirname(__file__), "examples", "no-text.png")+simple_txt = Path(__file__).parent / "samples" / "simple.txt"+simple_png = Path(__file__).parent / "samples" / "simple-noalpha.png"+simple_png2 = Path(__file__).parent / "examples" / "no-text.png"@override_settings(FILENAME_FORMAT="")@@ -198,13 +189,13 @@ class TestMigrateArchiveFiles(DirectoriesMixin, FileSystemAssertsMixin, TestMigrelse:self.assertIsNone(doc.archive_filename)- with open(source_path(doc), "rb") as f:+ with Path(source_path(doc)).open("rb") as f:original_checksum = hashlib.md5(f.read()).hexdigest()self.assertEqual(original_checksum, doc.checksum)if doc.archive_checksum:self.assertIsFile(archive_path_new(doc))- with open(archive_path_new(doc), "rb") as f:+ with archive_path_new(doc).open("rb") as f:archive_checksum = hashlib.md5(f.read()).hexdigest()self.assertEqual(archive_checksum, doc.archive_checksum)@@ -301,7 +292,7 @@ class TestMigrateArchiveFilesErrors(DirectoriesMixin, TestMigrations):"clash.pdf",simple_pdf,)- os.unlink(archive_path_old(doc))+ archive_path_old(doc).unlink()self.assertRaisesMessage(ValueError,@@ -494,13 +485,13 @@ class TestMigrateArchiveFilesBackwards(for doc in Document.objects.all():if doc.archive_checksum:self.assertIsFile(archive_path_old(doc))- with open(source_path(doc), "rb") as f:+ with Path(source_path(doc)).open("rb") as f:original_checksum = hashlib.md5(f.read()).hexdigest()self.assertEqual(original_checksum, doc.checksum)if doc.archive_checksum:self.assertIsFile(archive_path_old(doc))- with open(archive_path_old(doc), "rb") as f:+ with archive_path_old(doc).open("rb") as f:archive_checksum = hashlib.md5(f.read()).hexdigest()self.assertEqual(archive_checksum, doc.archive_checksum)