A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Fix: Apply unicode normalization to all paths and path components (#12993)
Trenton H8ed4bf201181
diff --git a/src/documents/templating/filepath.py b/src/documents/templating/filepath.pyindex 008f1d0a9..d827b7c89 100644--- a/src/documents/templating/filepath.py+++ b/src/documents/templating/filepath.py@@ -1,6 +1,7 @@import loggingimport osimport re+import unicodedatafrom collections.abc import Iterablefrom pathlib import PurePath@@ -36,10 +37,12 @@ class FilePathTemplate(Template):def clean_filepath(value: str) -> str:"""Clean up a filepath by:- 1. Removing newlines and carriage returns- 2. Removing extra spaces before and after forward slashes- 3. Preserving spaces in other parts of the path+ 1. Normalizing Unicode to NFC form to prevent byte-level mismatches+ 2. Removing newlines and carriage returns+ 3. Removing extra spaces before and after forward slashes+ 4. Preserving spaces in other parts of the path"""+ value = unicodedata.normalize("NFC", value)value = value.replace("\n", "").replace("\r", "")value = re.sub(r"\s*/\s*", "/", value)@@ -181,17 +184,17 @@ def get_basic_metadata_context("""return {"title": pathvalidate.sanitize_filename(- document.title,+ unicodedata.normalize("NFC", document.title),replacement_text="-",),"correspondent": pathvalidate.sanitize_filename(- document.correspondent.name,+ unicodedata.normalize("NFC", document.correspondent.name),replacement_text="-",)if document.correspondentelse no_value_default,"document_type": pathvalidate.sanitize_filename(- document.document_type.name,+ unicodedata.normalize("NFC", document.document_type.name),replacement_text="-",)if document.document_type@@ -202,7 +205,10 @@ def get_basic_metadata_context("owner_username": document.owner.usernameif document.ownerelse no_value_default,- "original_name": PurePath(document.original_filename).with_suffix("").name+ "original_name": unicodedata.normalize(+ "NFC",+ PurePath(document.original_filename).with_suffix("").name,+ )if document.original_filenameelse no_value_default,"doc_pk": f"{document.pk:07}",@@ -269,12 +275,12 @@ def get_tags_context(tags: Iterable[Tag]) -> dict[str, str | list[str]]:return {"tag_list": pathvalidate.sanitize_filename(",".join(- sorted(tag.name for tag in tags),+ sorted(unicodedata.normalize("NFC", tag.name) for tag in tags),),replacement_text="-",),# Assumed to be ordered, but a template could loop through to find what they want- "tag_name_list": [x.name for x in tags],+ "tag_name_list": [unicodedata.normalize("NFC", x.name) for x in tags],}@@ -301,7 +307,7 @@ def get_custom_fields_context(CustomField.FieldDataType.LONG_TEXT,}:value = pathvalidate.sanitize_filename(- field_instance.value,+ unicodedata.normalize("NFC", field_instance.value),replacement_text="-",)elif (@@ -310,10 +316,13 @@ def get_custom_fields_context():options = field_instance.field.extra_data["select_options"]value = pathvalidate.sanitize_filename(- next(- option["label"]- for option in options- if option["id"] == field_instance.value+ unicodedata.normalize(+ "NFC",+ next(+ option["label"]+ for option in options+ if option["id"] == field_instance.value+ ),),replacement_text="-",)@@ -321,7 +330,7 @@ def get_custom_fields_context(value = field_instance.valuefield_data["custom_fields"][pathvalidate.sanitize_filename(- field_instance.field.name,+ unicodedata.normalize("NFC", field_instance.field.name),replacement_text="-",)] = {diff --git a/src/documents/tests/test_api_post_document_nfc.py b/src/documents/tests/test_api_post_document_nfc.pynew file mode 100644index 000000000..5b3088b01--- /dev/null+++ b/src/documents/tests/test_api_post_document_nfc.py@@ -0,0 +1,95 @@+import unicodedata+from typing import TYPE_CHECKING+from unittest import mock++import celery.result+import pytest+from django.core.files.uploadedfile import SimpleUploadedFile++if TYPE_CHECKING:+ from documents.data_models import ConsumableDocument+ from documents.data_models import DocumentMetadataOverrides+++def consume_file_mock():+ with mock.patch("documents.tasks.consume_file.apply_async") as m:+ m.return_value = celery.result.AsyncResult(id="test-task-id")+ yield m+++def directories(tmp_path, settings, _media_settings):+ scratch = tmp_path / "scratch"+ scratch.mkdir()+ settings.SCRATCH_DIR = scratch+ return scratch+++class TestPostDocumentNFCNormalization:+ def test_nfd_filename_normalized_to_nfc(+ self,+ admin_client,+ consume_file_mock: mock.MagicMock,+ directories,+ ):+ """Uploaded file with NFD filename must have its name stored as NFC."""+ nfd = unicodedata.normalize("NFD", "Rechnung März.pdf")+ nfc = unicodedata.normalize("NFC", "Rechnung März.pdf")++ # Verify our test strings actually differ at the byte level+ assert nfd != nfc++ uploaded = SimpleUploadedFile(+ nfd,+ b"%PDF-1.4 test",+ content_type="application/pdf",+ )+ response = admin_client.post(+ "/api/documents/post_document/",+ {"document": uploaded},+ )++ assert response.status_code == 200++ task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]+ input_doc: ConsumableDocument = task_kwargs["input_doc"]+ overrides: DocumentMetadataOverrides = task_kwargs["overrides"]++ # The temp file on disk must have an NFC name+ assert input_doc.original_file.name == nfc, (+ f"Expected NFC filename {nfc!r}, got {input_doc.original_file.name!r}"+ )+ # The override filename stored for later use must also be NFC+ assert overrides.filename == nfc, (+ f"Expected NFC override filename {nfc!r}, got {overrides.filename!r}"+ )+ assert unicodedata.is_normalized("NFC", overrides.filename)++ def test_already_nfc_filename_unchanged(+ self,+ admin_client,+ consume_file_mock: mock.MagicMock,+ directories,+ ):+ """Uploaded file with already-NFC filename must pass through unchanged."""+ nfc = unicodedata.normalize("NFC", "Invoice_2024.pdf")++ uploaded = SimpleUploadedFile(+ nfc,+ b"%PDF-1.4 test",+ content_type="application/pdf",+ )+ response = admin_client.post(+ "/api/documents/post_document/",+ {"document": uploaded},+ )++ assert response.status_code == 200++ task_kwargs = consume_file_mock.call_args.kwargs["kwargs"]+ overrides: DocumentMetadataOverrides = task_kwargs["overrides"]++ assert overrides.filename == nfc+ assert unicodedata.is_normalized("NFC", overrides.filename)diff --git a/src/documents/tests/test_filepath_nfc.py b/src/documents/tests/test_filepath_nfc.pynew file mode 100644index 000000000..e1d3ef9a8--- /dev/null+++ b/src/documents/tests/test_filepath_nfc.py@@ -0,0 +1,187 @@+"""+Tests for NFC Unicode normalization in generate_filename / FilePathTemplate.render().++NFC `ü` (UTF-8: c3 bc) and NFD `ü` (UTF-8: 75 cc 88) are visually identical but+produce different byte sequences. On Linux (ext4, ZFS) these are distinct filenames.+All paths produced by the templating system must be NFC-normalized.+"""++import unicodedata++import pytest++from documents.file_handling import generate_filename+from documents.models import CustomField+from documents.models import CustomFieldInstance+from documents.tests.factories import CorrespondentFactory+from documents.tests.factories import DocumentFactory+from documents.tests.factories import StoragePathFactory+from documents.tests.factories import TagFactory+++class TestGenerateFilenameNFCNormalization:+ @pytest.mark.parametrize(+ "raw,display",+ [+ (unicodedata.normalize("NFD", "Gemüse"), "Gemüse"),+ (unicodedata.normalize("NFD", "Café"), "Café"),+ (unicodedata.normalize("NFD", "naïve"), "naïve"),+ ],+ )+ def test_nfd_title_normalized_to_nfc(self, settings, raw, display):+ """NFD title must produce NFC path bytes."""+ settings.FILENAME_FORMAT = "{{ title }}"+ nfc = unicodedata.normalize("NFC", display)+ assert raw != nfc # confirm byte-level difference++ doc = DocumentFactory(title=raw, mime_type="application/pdf")+ result = generate_filename(doc)++ assert str(result) == f"{nfc}.pdf"+ assert str(result).encode() == f"{nfc}.pdf".encode()++ def test_nfd_correspondent_normalized_to_nfc(self, settings):+ """NFD correspondent name must produce NFC path component."""+ settings.FILENAME_FORMAT = "{{ correspondent }}/{{ title }}"+ nfd = unicodedata.normalize("NFD", "Müller")+ nfc = unicodedata.normalize("NFC", "Müller")++ correspondent = CorrespondentFactory(name=nfd)+ doc = DocumentFactory(+ title="invoice",+ correspondent=correspondent,+ mime_type="application/pdf",+ )+ result = generate_filename(doc)++ assert str(result) == f"{nfc}/invoice.pdf"+ assert str(result).encode() == f"{nfc}/invoice.pdf".encode()++ def test_nfd_storage_path_normalized_to_nfc(self, settings):+ """NFD literal in StoragePath.path template must produce NFC path bytes."""+ settings.FILENAME_FORMAT = None+ nfd = unicodedata.normalize("NFD", "Büro")+ nfc = unicodedata.normalize("NFC", "Büro")++ # StoragePath.path is used directly as the format/template string.+ # Literal NFD characters in the template must survive rendering as NFC.+ sp = StoragePathFactory(path=f"{nfd}/{{{{ title }}}}")+ doc = DocumentFactory(title="doc", storage_path=sp, mime_type="application/pdf")+ result = generate_filename(doc)++ assert str(result).encode() == f"{nfc}/doc.pdf".encode()++ def test_nfd_raw_document_title_normalized_to_nfc(self, settings):+ """NFD title accessed via document.title (unsanitized context) must also be NFC."""+ settings.FILENAME_FORMAT = "{{ document.title }}"+ nfd = unicodedata.normalize("NFD", "Café")+ nfc = unicodedata.normalize("NFC", "Café")++ doc = DocumentFactory(title=nfd, mime_type="application/pdf")+ result = generate_filename(doc)++ assert str(result) == f"{nfc}.pdf"+ assert str(result).encode() == f"{nfc}.pdf".encode()+++class TestContextBuilderNFCNormalization:+ """+ Defense-in-depth: context builder functions must NFC-normalize string inputs+ before passing them to sanitize_filename(). Task 1 already normalizes the+ final rendered path via clean_filepath(), so these tests may already pass;+ they exist as regression guards for the context-builder layer.+ """++ def test_nfd_tag_name_normalized_in_tag_list(self, settings):+ """NFD tag name must appear as NFC bytes in the {{ tag_list }} shorthand."""+ settings.FILENAME_FORMAT = "{{ tag_list }}/{{ title }}"+ nfd = unicodedata.normalize("NFD", "Büro")+ nfc = unicodedata.normalize("NFC", "Büro")+ assert nfd != nfc # confirm they differ at byte level++ tag = TagFactory(name=nfd)+ doc = DocumentFactory(title="doc", mime_type="application/pdf")+ doc.tags.set([tag])++ result = generate_filename(doc)++ assert str(result).encode() == f"{nfc}/doc.pdf".encode()++ def test_nfd_original_name_normalized_to_nfc(self, settings):+ settings.FILENAME_FORMAT = "{{ original_name }}"+ nfd = unicodedata.normalize("NFD", "Rechnung März")+ nfc = unicodedata.normalize("NFC", "Rechnung März")++ doc = DocumentFactory(+ original_filename=f"{nfd}.pdf",+ mime_type="application/pdf",+ )+ result = generate_filename(doc)++ assert str(result).encode() == f"{nfc}.pdf".encode()++ def test_nfd_custom_field_string_value_normalized(self, settings):+ """NFD value in a STRING-type custom field must appear as NFC in the context."""+ settings.FILENAME_FORMAT = (+ "{{ custom_fields['Location']['value'] }}/{{ title }}"+ )+ nfd_value = unicodedata.normalize("NFD", "Düsseldorf")+ nfc_value = unicodedata.normalize("NFC", "Düsseldorf")+ assert nfd_value != nfc_value++ doc = DocumentFactory(title="report", mime_type="application/pdf")+ cf = CustomField.objects.create(+ name="Location",+ data_type=CustomField.FieldDataType.STRING,+ )+ CustomFieldInstance.objects.create(+ document=doc,+ field=cf,+ value_text=nfd_value,+ )++ result = generate_filename(doc)++ assert str(result).encode() == f"{nfc_value}/report.pdf".encode()++ def test_nfd_custom_field_name_normalized_as_key(self, settings):+ """NFD characters in a custom field name must appear as NFC in the context dict key."""+ nfd_name = unicodedata.normalize("NFD", "Größe")+ nfc_name = unicodedata.normalize("NFC", "Größe")+ assert nfd_name != nfc_name++ settings.FILENAME_FORMAT = f"{{% if custom_fields['{nfc_name}'] %}}{{{{ custom_fields['{nfc_name}']['value'] }}}}/{{{{ title }}}}{{% else %}}{{{{ title }}}}{{% endif %}}"++ doc = DocumentFactory(title="letter", mime_type="application/pdf")+ cf = CustomField.objects.create(+ name=nfd_name,+ data_type=CustomField.FieldDataType.STRING,+ )+ CustomFieldInstance.objects.create(+ document=doc,+ field=cf,+ value_text="Berlin",+ )++ result = generate_filename(doc)++ # If field name key is NFC-normalized, the template condition succeeds+ # and result is "Berlin/letter.pdf"; otherwise it falls back to "letter.pdf"+ assert str(result) == "Berlin/letter.pdf"++ def test_nfd_tag_name_list_normalized_to_nfc(self, settings):+ """NFD tag names in tag_name_list must appear as NFC bytes when iterated."""+ settings.FILENAME_FORMAT = (+ "{% for t in tag_name_list %}{{ t }}{% endfor %}/{{ title }}"+ )+ nfd = unicodedata.normalize("NFD", "Büro")+ nfc = unicodedata.normalize("NFC", "Büro")+ assert nfd != nfc # confirm byte-level difference++ doc = DocumentFactory(title="doc", mime_type="application/pdf")+ doc.tags.add(TagFactory(name=nfd))+ result = generate_filename(doc)++ assert str(result).encode() == f"{nfc}/doc.pdf".encode()diff --git a/src/documents/views.py b/src/documents/views.pyindex cbc4560d8..5ed6fdaf5 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -3126,6 +3126,7 @@ class PostDocumentView(GenericAPIView[Any]):serializer.is_valid(raise_exception=True)doc_name, doc_data = serializer.validated_data.get("document")+ doc_name = normalize("NFC", doc_name)correspondent_id = serializer.validated_data.get("correspondent")document_type_id = serializer.validated_data.get("document_type")storage_path_id = serializer.validated_data.get("storage_path")diff --git a/src/paperless_mail/mail.py b/src/paperless_mail/mail.pyindex d551cc8cd..acdc72703 100644--- a/src/paperless_mail/mail.py+++ b/src/paperless_mail/mail.py@@ -4,6 +4,7 @@ import loggingimport sslimport tempfileimport traceback+import unicodedatafrom datetime import datefrom datetime import timedeltafrom fnmatch import fnmatch@@ -496,10 +497,10 @@ class MailAccountHandler(LoggingMixin):rule: MailRule,) -> str | None:if rule.assign_title_from == MailRule.TitleSource.FROM_SUBJECT:- return message.subject+ return unicodedata.normalize("NFC", message.subject)elif rule.assign_title_from == MailRule.TitleSource.FROM_FILENAME:- return Path(att.filename).stem+ return unicodedata.normalize("NFC", Path(att.filename).stem)elif rule.assign_title_from == MailRule.TitleSource.NONE:return None@@ -866,7 +867,9 @@ class MailAccountHandler(LoggingMixin):),)- attachment_name = pathvalidate.sanitize_filename(att.filename)+ attachment_name = pathvalidate.sanitize_filename(+ unicodedata.normalize("NFC", att.filename),+ )if attachment_name:temp_filename = temp_dir / attachment_nameelse: # pragma: no cover@@ -882,7 +885,7 @@ class MailAccountHandler(LoggingMixin):)doc_overrides = DocumentMetadataOverrides(title=title,- filename=pathvalidate.sanitize_filename(att.filename),+ filename=attachment_name,correspondent_id=correspondent.id if correspondent else None,document_type_id=doc_type.id if doc_type else None,tag_ids=tag_ids,@@ -988,7 +991,9 @@ class MailAccountHandler(LoggingMixin):)doc_overrides = DocumentMetadataOverrides(title=message.subject,- filename=pathvalidate.sanitize_filename(f"{message.subject}.eml"),+ filename=pathvalidate.sanitize_filename(+ unicodedata.normalize("NFC", f"{message.subject}.eml"),+ ),correspondent_id=correspondent.id if correspondent else None,document_type_id=doc_type.id if doc_type else None,tag_ids=tag_ids,diff --git a/src/paperless_mail/tests/test_mail_nfc.py b/src/paperless_mail/tests/test_mail_nfc.pynew file mode 100644index 000000000..bfef06da9--- /dev/null+++ b/src/paperless_mail/tests/test_mail_nfc.py@@ -0,0 +1,182 @@+"""+Tests that mail attachment filenames and EML subject filenames are+normalized to NFC Unicode before being stored as document overrides.++Filenames from MIME headers can arrive in NFD form (e.g. from macOS Mail),+and must be normalized to NFC so filenames are consistent regardless of the+sending client.+"""++import unicodedata+from pathlib import Path+from unittest import mock++import pytest++from documents.tests.utils import remove_dirs+from documents.tests.utils import setup_directories+from paperless_mail.models import MailRule+from paperless_mail.tests.factories import MailAccountFactory+from paperless_mail.tests.test_mail import MessageBuilder+from paperless_mail.tests.test_mail import _AttachmentDef+from paperless_mail.tests.test_mail import fake_magic_from_buffer+++def directories(settings):+ dirs = setup_directories()+ yield dirs+ remove_dirs(dirs)+++def queue_consumption_tasks_mock():+ with mock.patch("paperless_mail.mail.queue_consumption_tasks") as m:+ yield m+++def mail_account(db):+ return MailAccountFactory()+++def attachment_rule(mail_account):+ rule = MailRule(+ name="attachment rule",+ account=mail_account,+ assign_title_from=MailRule.TitleSource.FROM_FILENAME,+ consumption_scope=MailRule.ConsumptionScope.ATTACHMENTS_ONLY,+ attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,+ )+ rule.save()+ return rule+++def eml_rule(mail_account):+ rule = MailRule(+ name="eml rule",+ account=mail_account,+ assign_title_from=MailRule.TitleSource.FROM_SUBJECT,+ consumption_scope=MailRule.ConsumptionScope.EML_ONLY,+ attachment_type=MailRule.AttachmentProcessing.ATTACHMENTS_ONLY,+ )+ rule.save()+ return rule+++def message_builder():+ return MessageBuilder()++[email protected]("paperless_mail.mail.magic.from_buffer", fake_magic_from_buffer)+class TestMailNFCNormalization:+ """Attachment filenames and EML subject filenames must be NFC-normalized."""++ def test_attachment_nfd_filename_normalized_to_nfc(+ self,+ directories,+ queue_consumption_tasks_mock,+ attachment_rule,+ mail_account_handler,+ message_builder,+ ):+ """Attachment filename arriving as NFD must be stored as NFC in both+ the overrides and the temp file written to disk.+ """+ nfd_filename = unicodedata.normalize("NFD", "Rechnung März.pdf")+ nfc_filename = unicodedata.normalize("NFC", "Rechnung März.pdf")++ # Confirm the fixture is actually NFD (not already NFC)+ assert unicodedata.is_normalized("NFD", nfd_filename)+ assert not unicodedata.is_normalized("NFC", nfd_filename)++ message = message_builder.create_message(+ subject="Test invoice",+ from_="[email protected]",+ attachments=[+ _AttachmentDef(filename=nfd_filename, content=b"%PDF-1.4 test"),+ ],+ )++ result = mail_account_handler._handle_message(message, attachment_rule)++ assert result == 1+ queue_consumption_tasks_mock.assert_called_once()++ call_kwargs = queue_consumption_tasks_mock.call_args.kwargs+ consume_tasks = call_kwargs["consume_tasks"]+ assert len(consume_tasks) == 1++ overrides = consume_tasks[0].kwargs["overrides"]+ assert overrides.filename == nfc_filename+ assert unicodedata.is_normalized("NFC", overrides.filename)+ assert unicodedata.is_normalized("NFC", overrides.title)++ input_doc = consume_tasks[0].kwargs["input_doc"]+ original_file = Path(input_doc.original_file)+ assert original_file.exists()+ assert original_file.name == nfc_filename++ def test_eml_subject_filename_nfc(+ self,+ directories,+ queue_consumption_tasks_mock,+ eml_rule,+ mail_account_handler,+ message_builder,+ ):+ """EML filename derived from subject arriving as NFD must be stored as NFC."""+ nfd_subject = unicodedata.normalize("NFD", "Rechnung März 2024")+ nfc_expected_filename = unicodedata.normalize("NFC", "Rechnung März 2024.eml")++ # Confirm the fixture is actually NFD+ assert unicodedata.is_normalized("NFD", nfd_subject)++ message = message_builder.create_message(+ subject=nfd_subject,+ from_="[email protected]",+ attachments=0,+ )++ mail_account_handler._handle_message(message, eml_rule)++ queue_consumption_tasks_mock.assert_called_once()++ call_kwargs = queue_consumption_tasks_mock.call_args.kwargs+ consume_tasks = call_kwargs["consume_tasks"]+ assert len(consume_tasks) == 1++ overrides = consume_tasks[0].kwargs["overrides"]+ assert overrides.filename == nfc_expected_filename+ assert unicodedata.is_normalized("NFC", overrides.filename)++ def test_already_nfc_attachment_filename_unchanged(+ self,+ directories,+ queue_consumption_tasks_mock,+ attachment_rule,+ mail_account_handler,+ message_builder,+ ):+ """An attachment filename already in NFC must pass through unchanged."""+ nfc_filename = "Invoice_2024.pdf"+ assert unicodedata.is_normalized("NFC", nfc_filename)++ message = message_builder.create_message(+ subject="Invoice",+ from_="[email protected]",+ attachments=[+ _AttachmentDef(filename=nfc_filename, content=b"%PDF-1.4 test"),+ ],+ )++ mail_account_handler._handle_message(message, attachment_rule)++ call_kwargs = queue_consumption_tasks_mock.call_args.kwargs+ consume_tasks = call_kwargs["consume_tasks"]+ overrides = consume_tasks[0].kwargs["overrides"]+ assert overrides.filename == nfc_filename