A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Chore: Drop old signal and unneeded apps, transition to parser registry instead (#12405)
Trenton H701735f6e55d
diff --git a/.mypy-baseline.txt b/.mypy-baseline.txtindex 2700bfc71..61ffe9c10 100644--- a/.mypy-baseline.txt+++ b/.mypy-baseline.txt@@ -2437,17 +2437,3 @@ src/paperless_tesseract/tests/test_parser_custom_settings.py:0: error: Item "Nonsrc/paperless_tesseract/tests/test_parser_custom_settings.py:0: error: Item "None" of "ApplicationConfiguration | None" has no attribute "unpaper_clean" [union-attr]src/paperless_tesseract/tests/test_parser_custom_settings.py:0: error: Item "None" of "ApplicationConfiguration | None" has no attribute "unpaper_clean" [union-attr]src/paperless_tesseract/tests/test_parser_custom_settings.py:0: error: Item "None" of "ApplicationConfiguration | None" has no attribute "user_args" [union-attr]-src/paperless_text/parsers.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def]-src/paperless_text/parsers.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def]-src/paperless_text/parsers.py:0: error: Incompatible types in assignment (expression has type "str", variable has type "None") [assignment]-src/paperless_text/signals.py:0: error: Function is missing a type annotation [no-untyped-def]-src/paperless_text/signals.py:0: error: Function is missing a type annotation [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Argument 1 to "make_thumbnail_from_pdf" has incompatible type "None"; expected "Path" [arg-type]-src/paperless_tika/parsers.py:0: error: Function is missing a return type annotation [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Function is missing a type annotation [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Function is missing a type annotation [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Function is missing a type annotation for one or more arguments [no-untyped-def]-src/paperless_tika/parsers.py:0: error: Incompatible types in assignment (expression has type "str | None", variable has type "None") [assignment]-src/paperless_tika/signals.py:0: error: Function is missing a type annotation [no-untyped-def]-src/paperless_tika/signals.py:0: error: Function is missing a type annotation [no-untyped-def]diff --git a/pyproject.toml b/pyproject.tomlindex f2a20ac47..ee89ae4dd 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -269,10 +269,6 @@ testpaths = ["src/documents/tests/","src/paperless/tests/","src/paperless_mail/tests/",- "src/paperless_tesseract/tests/",- "src/paperless_tika/tests",- "src/paperless_text/tests/",- "src/paperless_remote/tests/","src/paperless_ai/tests",]diff --git a/src/documents/checks.py b/src/documents/checks.pyindex b6e9e90fc..0867ef403 100644--- a/src/documents/checks.py+++ b/src/documents/checks.py@@ -3,25 +3,20 @@ from django.core.checks import Errorfrom django.core.checks import Warningfrom django.core.checks import register-from documents.signals import document_consumer_declarationfrom documents.templating.utils import convert_format_str_to_template_format+from paperless.parsers.registry import get_parser_registry@register()def parser_check(app_configs, **kwargs):- parsers = []- for response in document_consumer_declaration.send(None):- parsers.append(response[1])-- if len(parsers) == 0:+ if not get_parser_registry().all_parsers():return [Error("No parsers found. This is a bug. The consumer won't be ""able to consume any documents without parsers.",),]- else:- return []+ return []@register()diff --git a/src/documents/consumer.py b/src/documents/consumer.pyindex 81d9eb456..809d6c647 100644--- a/src/documents/consumer.py+++ b/src/documents/consumer.py@@ -32,9 +32,7 @@ from documents.models import DocumentTypefrom documents.models import StoragePathfrom documents.models import Tagfrom documents.models import WorkflowTrigger-from documents.parsers import DocumentParserfrom documents.parsers import ParseError-from documents.parsers import get_parser_class_for_mime_typefrom documents.permissions import set_permissions_for_objectfrom documents.plugins.base import AlwaysRunPluginMixinfrom documents.plugins.base import ConsumeTaskPlugin@@ -52,40 +50,12 @@ from documents.utils import copy_basic_file_statsfrom documents.utils import copy_file_with_basic_statsfrom documents.utils import run_subprocessfrom paperless.parsers import ParserContext-from paperless.parsers.mail import MailDocumentParser-from paperless.parsers.remote import RemoteDocumentParser-from paperless.parsers.tesseract import RasterisedDocumentParser-from paperless.parsers.text import TextDocumentParser-from paperless.parsers.tika import TikaDocumentParser+from paperless.parsers import ParserProtocol+from paperless.parsers.registry import get_parser_registryLOGGING_NAME: Final[str] = "paperless.consumer"-def _parser_cleanup(parser: DocumentParser) -> None:- """- Call cleanup on a parser, handling the new-style context-manager parsers.-- New-style parsers (e.g. TextDocumentParser) use __exit__ for teardown- instead of a cleanup() method. This shim will be removed once all existing parsers- have switched to the new style and this consumer is updated to use it-- TODO(stumpylog): Remove me in the future- """- if isinstance(- parser,- (- MailDocumentParser,- RasterisedDocumentParser,- RemoteDocumentParser,- TextDocumentParser,- TikaDocumentParser,- ),- ):- parser.__exit__(None, None, None)- else:- parser.cleanup()--class WorkflowTriggerPlugin(NoCleanupPluginMixin,NoSetupPluginMixin,@@ -422,8 +392,12 @@ class ConsumerPlugin(self.log.error(f"Error attempting to clean PDF: {e}")# Based on the mime type, get the parser for that type- parser_class: type[DocumentParser] | None = get_parser_class_for_mime_type(- mime_type,+ parser_class: type[ParserProtocol] | None = (+ get_parser_registry().get_parser_for_file(+ mime_type,+ self.filename,+ self.working_copy,+ ))if not parser_class:tempdir.cleanup()@@ -446,313 +420,275 @@ class ConsumerPlugin(tempdir.cleanup()raise- def progress_callback(- current_progress,- max_progress,- ) -> None: # pragma: no cover- # recalculate progress to be within 20 and 80- p = int((current_progress / max_progress) * 50 + 20)- self._send_progress(p, 100, ProgressStatusOptions.WORKING)-# This doesn't parse the document yet, but gives us a parser.+ with parser_class() as document_parser:+ document_parser.configure(+ ParserContext(mailrule_id=self.input_doc.mailrule_id),+ )- document_parser: DocumentParser = parser_class(- self.logging_group,- progress_callback=progress_callback,- )-- parser_is_new_style = isinstance(- document_parser,- (- MailDocumentParser,- RasterisedDocumentParser,- RemoteDocumentParser,- TextDocumentParser,- TikaDocumentParser,- ),- )-- # New-style parsers use __enter__/__exit__ for resource management.- # _parser_cleanup (below) handles __exit__; call __enter__ here.- # TODO(stumpylog): Remove me in the future- if parser_is_new_style:- document_parser.__enter__()-- self.log.debug(f"Parser: {type(document_parser).__name__}")-- # Parse the document. This may take some time.+ self.log.debug(f"Parser: {document_parser.name} v{document_parser.version}")- text = None- date = None- thumbnail = None- archive_path = None- page_count = None+ # Parse the document. This may take some time.- try:- self._send_progress(- 20,- 100,- ProgressStatusOptions.WORKING,- ConsumerStatusShortMessage.PARSING_DOCUMENT,- )- self.log.debug(f"Parsing {self.filename}...")+ text = None+ date = None+ thumbnail = None+ archive_path = None+ page_count = None- # TODO(stumpylog): Remove me in the future when all parsers use new protocol- if parser_is_new_style:- document_parser.configure(- ParserContext(mailrule_id=self.input_doc.mailrule_id),+ try:+ self._send_progress(+ 20,+ 100,+ ProgressStatusOptions.WORKING,+ ConsumerStatusShortMessage.PARSING_DOCUMENT,)- # TODO(stumpylog): Remove me in the future+ self.log.debug(f"Parsing {self.filename}...")+document_parser.parse(self.working_copy, mime_type)- else:- document_parser.parse(self.working_copy, mime_type, self.filename)- self.log.debug(f"Generating thumbnail for {self.filename}...")- self._send_progress(- 70,- 100,- ProgressStatusOptions.WORKING,- ConsumerStatusShortMessage.GENERATING_THUMBNAIL,- )- # TODO(stumpylog): Remove me in the future when all parsers use new protocol- if parser_is_new_style:+ self.log.debug(f"Generating thumbnail for {self.filename}...")+ self._send_progress(+ 70,+ 100,+ ProgressStatusOptions.WORKING,+ ConsumerStatusShortMessage.GENERATING_THUMBNAIL,+ )thumbnail = document_parser.get_thumbnail(self.working_copy, mime_type)- else:- thumbnail = document_parser.get_thumbnail(++ text = document_parser.get_text()+ date = document_parser.get_date()+ if date is None:+ self._send_progress(+ 90,+ 100,+ ProgressStatusOptions.WORKING,+ ConsumerStatusShortMessage.PARSE_DATE,+ )+ with get_date_parser() as date_parser:+ date = next(date_parser.parse(self.filename, text), None)+ archive_path = document_parser.get_archive_path()+ page_count = document_parser.get_page_count(self.working_copy,mime_type,- self.filename,)- text = document_parser.get_text()- date = document_parser.get_date()- if date is None:- self._send_progress(- 90,- 100,- ProgressStatusOptions.WORKING,- ConsumerStatusShortMessage.PARSE_DATE,+ except ParseError as e:+ if tempdir:+ tempdir.cleanup()+ self._fail(+ str(e),+ f"Error occurred while consuming document {self.filename}: {e}",+ exc_info=True,+ exception=e,+ )+ except Exception as e:+ if tempdir:+ tempdir.cleanup()+ self._fail(+ str(e),+ f"Unexpected error while consuming document {self.filename}: {e}",+ exc_info=True,+ exception=e,)- with get_date_parser() as date_parser:- date = next(date_parser.parse(self.filename, text), None)- archive_path = document_parser.get_archive_path()- page_count = document_parser.get_page_count(self.working_copy, mime_type)-- except ParseError as e:- _parser_cleanup(document_parser)- if tempdir:- tempdir.cleanup()- self._fail(- str(e),- f"Error occurred while consuming document {self.filename}: {e}",- exc_info=True,- exception=e,- )- except Exception as e:- _parser_cleanup(document_parser)- if tempdir:- tempdir.cleanup()- self._fail(- str(e),- f"Unexpected error while consuming document {self.filename}: {e}",- exc_info=True,- exception=e,- )- # Prepare the document classifier.+ # Prepare the document classifier.- # TODO: I don't really like to do this here, but this way we avoid- # reloading the classifier multiple times, since there are multiple- # post-consume hooks that all require the classifier.+ # TODO: I don't really like to do this here, but this way we avoid+ # reloading the classifier multiple times, since there are multiple+ # post-consume hooks that all require the classifier.- classifier = load_classifier()+ classifier = load_classifier()- self._send_progress(- 95,- 100,- ProgressStatusOptions.WORKING,- ConsumerStatusShortMessage.SAVE_DOCUMENT,- )- # now that everything is done, we can start to store the document- # in the system. This will be a transaction and reasonably fast.- try:- with transaction.atomic():- # store the document.- if self.input_doc.root_document_id:- # If this is a new version of an existing document, we need- # to make sure we're not creating a new document, but updating- # the existing one.- root_doc = Document.objects.get(- pk=self.input_doc.root_document_id,- )- original_document = self._create_version_from_root(- root_doc,- text=text,- page_count=page_count,- mime_type=mime_type,- )- actor = None-- # Save the new version, potentially creating an audit log entry for the version addition if enabled.- if (- settings.AUDIT_LOG_ENABLED- and self.metadata.actor_id is not None- ):- actor = User.objects.filter(pk=self.metadata.actor_id).first()- if actor is not None:- from auditlog.context import ( # type: ignore[import-untyped]- set_actor,- )+ self._send_progress(+ 95,+ 100,+ ProgressStatusOptions.WORKING,+ ConsumerStatusShortMessage.SAVE_DOCUMENT,+ )+ # now that everything is done, we can start to store the document+ # in the system. This will be a transaction and reasonably fast.+ try:+ with transaction.atomic():+ # store the document.+ if self.input_doc.root_document_id:+ # If this is a new version of an existing document, we need+ # to make sure we're not creating a new document, but updating+ # the existing one.+ root_doc = Document.objects.get(+ pk=self.input_doc.root_document_id,+ )+ original_document = self._create_version_from_root(+ root_doc,+ text=text,+ page_count=page_count,+ mime_type=mime_type,+ )+ actor = None- with set_actor(actor):+ # Save the new version, potentially creating an audit log entry for the version addition if enabled.+ if (+ settings.AUDIT_LOG_ENABLED+ and self.metadata.actor_id is not None+ ):+ actor = User.objects.filter(+ pk=self.metadata.actor_id,+ ).first()+ if actor is not None:+ from auditlog.context import ( # type: ignore[import-untyped]+ set_actor,+ )++ with set_actor(actor):+ original_document.save()+ else:original_document.save()else:original_document.save()- else:- original_document.save()- # Create a log entry for the version addition, if enabled- if settings.AUDIT_LOG_ENABLED:- from auditlog.models import ( # type: ignore[import-untyped]- LogEntry,- )+ # Create a log entry for the version addition, if enabled+ if settings.AUDIT_LOG_ENABLED:+ from auditlog.models import ( # type: ignore[import-untyped]+ LogEntry,+ )- LogEntry.objects.log_create(- instance=root_doc,- changes={- "Version Added": ["None", original_document.id],- },- action=LogEntry.Action.UPDATE,- actor=actor,- additional_data={- "reason": "Version added",- "version_id": original_document.id,- },+ LogEntry.objects.log_create(+ instance=root_doc,+ changes={+ "Version Added": ["None", original_document.id],+ },+ action=LogEntry.Action.UPDATE,+ actor=actor,+ additional_data={+ "reason": "Version added",+ "version_id": original_document.id,+ },+ )+ document = original_document+ else:+ document = self._store(+ text=text,+ date=date,+ page_count=page_count,+ mime_type=mime_type,)- document = original_document- else:- document = self._store(- text=text,- date=date,- page_count=page_count,- mime_type=mime_type,- )- # If we get here, it was successful. Proceed with post-consume- # hooks. If they fail, nothing will get changed.-- document_consumption_finished.send(- sender=self.__class__,- document=document,- logging_group=self.logging_group,- classifier=classifier,- original_file=self.unmodified_original- if self.unmodified_original- else self.working_copy,- )+ # If we get here, it was successful. Proceed with post-consume+ # hooks. If they fail, nothing will get changed.- # After everything is in the database, copy the files into- # place. If this fails, we'll also rollback the transaction.- with FileLock(settings.MEDIA_LOCK):- generated_filename = generate_unique_filename(document)- if (- len(str(generated_filename))- > Document.MAX_STORED_FILENAME_LENGTH- ):- self.log.warning(- "Generated source filename exceeds db path limit, falling back to default naming",- )- generated_filename = generate_filename(- document,- use_format=False,- )- document.filename = generated_filename- create_source_path_directory(document.source_path)-- self._write(- self.unmodified_original- if self.unmodified_original is not None+ document_consumption_finished.send(+ sender=self.__class__,+ document=document,+ logging_group=self.logging_group,+ classifier=classifier,+ original_file=self.unmodified_original+ if self.unmodified_originalelse self.working_copy,- document.source_path,)- self._write(- thumbnail,- document.thumbnail_path,- )-- if archive_path and Path(archive_path).is_file():- generated_archive_filename = generate_unique_filename(- document,- archive_filename=True,- )+ # After everything is in the database, copy the files into+ # place. If this fails, we'll also rollback the transaction.+ with FileLock(settings.MEDIA_LOCK):+ generated_filename = generate_unique_filename(document)if (- len(str(generated_archive_filename))+ len(str(generated_filename))> Document.MAX_STORED_FILENAME_LENGTH):self.log.warning(- "Generated archive filename exceeds db path limit, falling back to default naming",+ "Generated source filename exceeds db path limit, falling back to default naming",)- generated_archive_filename = generate_filename(+ generated_filename = generate_filename(document,- archive_filename=True,use_format=False,)- document.archive_filename = generated_archive_filename- create_source_path_directory(document.archive_path)+ document.filename = generated_filename+ create_source_path_directory(document.source_path)++ self._write(+ self.unmodified_original+ if self.unmodified_original is not None+ else self.working_copy,+ document.source_path,+ )+self._write(- archive_path,- document.archive_path,+ thumbnail,+ document.thumbnail_path,)- with Path(archive_path).open("rb") as f:- document.archive_checksum = hashlib.md5(- f.read(),- ).hexdigest()+ if archive_path and Path(archive_path).is_file():+ generated_archive_filename = generate_unique_filename(+ document,+ archive_filename=True,+ )+ if (+ len(str(generated_archive_filename))+ > Document.MAX_STORED_FILENAME_LENGTH+ ):+ self.log.warning(+ "Generated archive filename exceeds db path limit, falling back to default naming",+ )+ generated_archive_filename = generate_filename(+ document,+ archive_filename=True,+ use_format=False,+ )+ document.archive_filename = generated_archive_filename+ create_source_path_directory(document.archive_path)+ self._write(+ archive_path,+ document.archive_path,+ )- # Don't save with the lock active. Saving will cause the file- # renaming logic to acquire the lock as well.- # This triggers things like file renaming- document.save()+ with Path(archive_path).open("rb") as f:+ document.archive_checksum = hashlib.md5(+ f.read(),+ ).hexdigest()- if document.root_document_id:- document_updated.send(- sender=self.__class__,- document=document.root_document,- )+ # Don't save with the lock active. Saving will cause the file+ # renaming logic to acquire the lock as well.+ # This triggers things like file renaming+ document.save()- # Delete the file only if it was successfully consumed- self.log.debug(f"Deleting original file {self.input_doc.original_file}")- self.input_doc.original_file.unlink()- self.log.debug(f"Deleting working copy {self.working_copy}")- self.working_copy.unlink()- if self.unmodified_original is not None: # pragma: no cover+ if document.root_document_id:+ document_updated.send(+ sender=self.__class__,+ document=document.root_document,+ )++ # Delete the file only if it was successfully consumedself.log.debug(- f"Deleting unmodified original file {self.unmodified_original}",+ f"Deleting original file {self.input_doc.original_file}",)- self.unmodified_original.unlink()+ self.input_doc.original_file.unlink()+ self.log.debug(f"Deleting working copy {self.working_copy}")+ self.working_copy.unlink()+ if self.unmodified_original is not None: # pragma: no cover+ self.log.debug(+ f"Deleting unmodified original file {self.unmodified_original}",+ )+ self.unmodified_original.unlink()- # https://github.com/jonaswinkler/paperless-ng/discussions/1037- shadow_file = (- Path(self.input_doc.original_file).parent- / f"._{Path(self.input_doc.original_file).name}"- )+ # https://github.com/jonaswinkler/paperless-ng/discussions/1037+ shadow_file = (+ Path(self.input_doc.original_file).parent+ / f"._{Path(self.input_doc.original_file).name}"+ )- if Path(shadow_file).is_file():- self.log.debug(f"Deleting shadow file {shadow_file}")- Path(shadow_file).unlink()+ if Path(shadow_file).is_file():+ self.log.debug(f"Deleting shadow file {shadow_file}")+ Path(shadow_file).unlink()- except Exception as e:- self._fail(- str(e),- f"The following error occurred while storing document "- f"{self.filename} after parsing: {e}",- exc_info=True,- exception=e,- )- finally:- _parser_cleanup(document_parser)- tempdir.cleanup()+ except Exception as e:+ self._fail(+ str(e),+ f"The following error occurred while storing document "+ f"{self.filename} after parsing: {e}",+ exc_info=True,+ exception=e,+ )+ finally:+ tempdir.cleanup()self.run_post_consume_script(document)diff --git a/src/documents/management/commands/document_thumbnails.py b/src/documents/management/commands/document_thumbnails.pyindex 1756f8754..3d779ae18 100644--- a/src/documents/management/commands/document_thumbnails.py+++ b/src/documents/management/commands/document_thumbnails.py@@ -3,19 +3,18 @@ import shutilfrom documents.management.commands.base import PaperlessCommandfrom documents.models import Document-from documents.parsers import get_parser_class_for_mime_type-from paperless.parsers.mail import MailDocumentParser-from paperless.parsers.remote import RemoteDocumentParser-from paperless.parsers.tesseract import RasterisedDocumentParser-from paperless.parsers.text import TextDocumentParser-from paperless.parsers.tika import TikaDocumentParser+from paperless.parsers.registry import get_parser_registrylogger = logging.getLogger("paperless.management.thumbnails")def _process_document(doc_id: int) -> None:document: Document = Document.objects.get(id=doc_id)- parser_class = get_parser_class_for_mime_type(document.mime_type)+ parser_class = get_parser_registry().get_parser_for_file(+ document.mime_type,+ document.original_filename or "",+ document.source_path,+ )if parser_class is None:logger.warning(@@ -25,40 +24,9 @@ def _process_document(doc_id: int) -> None:)return- parser = parser_class(logging_group=None)-- parser_is_new_style = isinstance(- parser,- (- MailDocumentParser,- RasterisedDocumentParser,- RemoteDocumentParser,- TextDocumentParser,- TikaDocumentParser,- ),- )-- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if parser_is_new_style:- parser.__enter__()-- try:- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if parser_is_new_style:- thumb = parser.get_thumbnail(document.source_path, document.mime_type)- else:- thumb = parser.get_thumbnail(- document.source_path,- document.mime_type,- document.get_public_filename(),- )+ with parser_class() as parser:+ thumb = parser.get_thumbnail(document.source_path, document.mime_type)shutil.move(thumb, document.thumbnail_path)- finally:- # TODO(stumpylog): Cleanup once all parsers are handled- if parser_is_new_style:- parser.__exit__(None, None, None)- else:- parser.cleanup()class Command(PaperlessCommand):diff --git a/src/documents/parsers.py b/src/documents/parsers.pyindex 372cf0491..69ee4e285 100644--- a/src/documents/parsers.py+++ b/src/documents/parsers.py@@ -3,84 +3,47 @@ from __future__ import annotationsimport loggingimport mimetypesimport os-import reimport shutilimport subprocessimport tempfile-from functools import lru_cachefrom pathlib import Pathfrom typing import TYPE_CHECKINGfrom django.conf import settingsfrom documents.loggers import LoggingMixin-from documents.signals import document_consumer_declarationfrom documents.utils import copy_file_with_basic_statsfrom documents.utils import run_subprocess+from paperless.parsers.registry import get_parser_registryif TYPE_CHECKING:import datetime-# This regular expression will try to find dates in the document at-# hand and will match the following formats:-# - XX.YY.ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - XX/YY/ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - XX-YY-ZZZZ with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - ZZZZ.XX.YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - ZZZZ/XX/YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - ZZZZ-XX-YY with XX + YY being 1 or 2 and ZZZZ being 2 or 4 digits-# - XX. MONTH ZZZZ with XX being 1 or 2 and ZZZZ being 2 or 4 digits-# - MONTH ZZZZ, with ZZZZ being 4 digits-# - MONTH XX, ZZZZ with XX being 1 or 2 and ZZZZ being 4 digits-# - XX MON ZZZZ with XX being 1 or 2 and ZZZZ being 4 digits. MONTH is 3 letters-# - XXPP MONTH ZZZZ with XX being 1 or 2 and PP being 2 letters and ZZZZ being 4 digits--# TODO: isn't there a date parsing library for this?--DATE_REGEX = re.compile(- r"(\b|(?!=([_-])))(\d{1,2})[\.\/-](\d{1,2})[\.\/-](\d{4}|\d{2})(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))(\d{4}|\d{2})[\.\/-](\d{1,2})[\.\/-](\d{1,2})(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))(\d{1,2}[\. ]+[a-zéûäëčžúřěáíóńźçŞğü]{3,9} \d{4}|[a-zéûäëčžúřěáíóńźçŞğü]{3,9} \d{1,2}, \d{4})(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))([^\W\d_]{3,9} \d{1,2}, (\d{4}))(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))([^\W\d_]{3,9} \d{4})(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))(\d{1,2}[^ 0-9]{2}[\. ]+[^ ]{3,9}[ \.\/-]\d{4})(\b|(?=([_-])))|"- r"(\b|(?!=([_-])))(\b\d{1,2}[ \.\/-][a-zéûäëčžúřěáíóńźçŞğü]{3}[ \.\/-]\d{4})(\b|(?=([_-])))",- re.IGNORECASE,-)--logger = logging.getLogger("paperless.parsing")-@lru_cache(maxsize=8)def is_mime_type_supported(mime_type: str) -> bool:"""Returns True if the mime type is supported, False otherwise"""- return get_parser_class_for_mime_type(mime_type) is not None+ return get_parser_registry().get_parser_for_file(mime_type, "") is not None-@lru_cache(maxsize=8)def get_default_file_extension(mime_type: str) -> str:"""Returns the default file extension for a mimetype, oran empty string if it could not be determined"""- for response in document_consumer_declaration.send(None):- parser_declaration = response[1]- supported_mime_types = parser_declaration["mime_types"]-- if mime_type in supported_mime_types:- return supported_mime_types[mime_type]+ parser_class = get_parser_registry().get_parser_for_file(mime_type, "")+ if parser_class is not None:+ supported = parser_class.supported_mime_types()+ if mime_type in supported:+ return supported[mime_type]ext = mimetypes.guess_extension(mime_type)- if ext:- return ext- else:- return ""+ return ext if ext else ""-@lru_cache(maxsize=8)def is_file_ext_supported(ext: str) -> bool:"""Returns True if the file extension is supported, False otherwise@@ -94,44 +57,17 @@ def is_file_ext_supported(ext: str) -> bool:def get_supported_file_extensions() -> set[str]:extensions = set()- for response in document_consumer_declaration.send(None):- parser_declaration = response[1]- supported_mime_types = parser_declaration["mime_types"]-- for mime_type in supported_mime_types:+ for parser_class in get_parser_registry().all_parsers():+ for mime_type, ext in parser_class.supported_mime_types().items():extensions.update(mimetypes.guess_all_extensions(mime_type))# Python's stdlib might be behind, so also add what the parser# says is the default extension# This makes image/webp supported on Python < 3.11- extensions.add(supported_mime_types[mime_type])+ extensions.add(ext)return extensions-def get_parser_class_for_mime_type(mime_type: str) -> type[DocumentParser] | None:- """- Returns the best parser (by weight) for the given mimetype or- None if no parser exists- """-- options = []-- for response in document_consumer_declaration.send(None):- parser_declaration = response[1]- supported_mime_types = parser_declaration["mime_types"]-- if mime_type in supported_mime_types:- options.append(parser_declaration)-- if not options:- return None-- best_parser = sorted(options, key=lambda _: _["weight"], reverse=True)[0]-- # Return the parser with the highest weight.- return best_parser["parser"]--def run_convert(input_file,output_file,diff --git a/src/documents/signals/__init__.py b/src/documents/signals/__init__.pyindex fbb55d9fe..864fec09f 100644--- a/src/documents/signals/__init__.py+++ b/src/documents/signals/__init__.py@@ -2,5 +2,4 @@ from django.dispatch import Signaldocument_consumption_started = Signal()document_consumption_finished = Signal()-document_consumer_declaration = Signal()document_updated = Signal()diff --git a/src/documents/tasks.py b/src/documents/tasks.pyindex a8ca0cc5f..751990c62 100644--- a/src/documents/tasks.py+++ b/src/documents/tasks.py@@ -52,8 +52,6 @@ from documents.models import StoragePathfrom documents.models import Tagfrom documents.models import WorkflowRunfrom documents.models import WorkflowTrigger-from documents.parsers import DocumentParser-from documents.parsers import get_parser_class_for_mime_typefrom documents.plugins.base import ConsumeTaskPluginfrom documents.plugins.base import ProgressManagerfrom documents.plugins.base import StopConsumeTaskError@@ -66,11 +64,7 @@ from documents.signals.handlers import send_websocket_document_updatedfrom documents.workflows.utils import get_workflows_for_triggerfrom paperless.config import AIConfigfrom paperless.parsers import ParserContext-from paperless.parsers.mail import MailDocumentParser-from paperless.parsers.remote import RemoteDocumentParser-from paperless.parsers.tesseract import RasterisedDocumentParser-from paperless.parsers.text import TextDocumentParser-from paperless.parsers.tika import TikaDocumentParser+from paperless.parsers.registry import get_parser_registryfrom paperless_ai.indexing import llm_index_add_or_update_documentfrom paperless_ai.indexing import llm_index_remove_documentfrom paperless_ai.indexing import update_llm_index@@ -310,8 +304,10 @@ def update_document_content_maybe_archive_file(document_id) -> None:mime_type = document.mime_type- parser_class: type[DocumentParser] | None = get_parser_class_for_mime_type(+ parser_class = get_parser_registry().get_parser_for_file(mime_type,+ document.original_filename or "",+ document.source_path,)if not parser_class:@@ -321,138 +317,92 @@ def update_document_content_maybe_archive_file(document_id) -> None:)return- parser: DocumentParser = parser_class(logging_group=uuid.uuid4())-- parser_is_new_style = isinstance(- parser,- (- MailDocumentParser,- RasterisedDocumentParser,- RemoteDocumentParser,- TextDocumentParser,- TikaDocumentParser,- ),- )-- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if parser_is_new_style:- parser.__enter__()+ with parser_class() as parser:+ parser.configure(ParserContext())- try:- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if parser_is_new_style:- parser.configure(ParserContext())+ try:parser.parse(document.source_path, mime_type)- else:- parser.parse(- document.source_path,- mime_type,- document.get_public_filename(),- )- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if parser_is_new_style:thumbnail = parser.get_thumbnail(document.source_path, mime_type)- else:- thumbnail = parser.get_thumbnail(- document.source_path,- mime_type,- document.get_public_filename(),- )- with transaction.atomic():- oldDocument = Document.objects.get(pk=document.pk)- if parser.get_archive_path():- with Path(parser.get_archive_path()).open("rb") as f:- checksum = hashlib.md5(f.read()).hexdigest()- # I'm going to save first so that in case the file move- # fails, the database is rolled back.- # We also don't use save() since that triggers the filehandling- # logic, and we don't want that yet (file not yet in place)- document.archive_filename = generate_unique_filename(- document,- archive_filename=True,- )- Document.objects.filter(pk=document.pk).update(- archive_checksum=checksum,- content=parser.get_text(),- archive_filename=document.archive_filename,- )- newDocument = Document.objects.get(pk=document.pk)- if settings.AUDIT_LOG_ENABLED:- LogEntry.objects.log_create(- instance=oldDocument,- changes={- "content": [oldDocument.content, newDocument.content],- "archive_checksum": [- oldDocument.archive_checksum,- newDocument.archive_checksum,- ],- "archive_filename": [- oldDocument.archive_filename,- newDocument.archive_filename,- ],- },- additional_data={- "reason": "Update document content",- },- action=LogEntry.Action.UPDATE,+ with transaction.atomic():+ oldDocument = Document.objects.get(pk=document.pk)+ if parser.get_archive_path():+ with Path(parser.get_archive_path()).open("rb") as f:+ checksum = hashlib.md5(f.read()).hexdigest()+ # I'm going to save first so that in case the file move+ # fails, the database is rolled back.+ # We also don't use save() since that triggers the filehandling+ # logic, and we don't want that yet (file not yet in place)+ document.archive_filename = generate_unique_filename(+ document,+ archive_filename=True,)- else:- Document.objects.filter(pk=document.pk).update(- content=parser.get_text(),- )-- if settings.AUDIT_LOG_ENABLED:- LogEntry.objects.log_create(- instance=oldDocument,- changes={- "content": [oldDocument.content, parser.get_text()],- },- additional_data={- "reason": "Update document content",- },- action=LogEntry.Action.UPDATE,+ Document.objects.filter(pk=document.pk).update(+ archive_checksum=checksum,+ content=parser.get_text(),+ archive_filename=document.archive_filename,+ )+ newDocument = Document.objects.get(pk=document.pk)+ if settings.AUDIT_LOG_ENABLED:+ LogEntry.objects.log_create(+ instance=oldDocument,+ changes={+ "content": [oldDocument.content, newDocument.content],+ "archive_checksum": [+ oldDocument.archive_checksum,+ newDocument.archive_checksum,+ ],+ "archive_filename": [+ oldDocument.archive_filename,+ newDocument.archive_filename,+ ],+ },+ additional_data={+ "reason": "Update document content",+ },+ action=LogEntry.Action.UPDATE,+ )+ else:+ Document.objects.filter(pk=document.pk).update(+ content=parser.get_text(),)- with FileLock(settings.MEDIA_LOCK):- if parser.get_archive_path():- create_source_path_directory(document.archive_path)- shutil.move(parser.get_archive_path(), document.archive_path)- shutil.move(thumbnail, document.thumbnail_path)+ if settings.AUDIT_LOG_ENABLED:+ LogEntry.objects.log_create(+ instance=oldDocument,+ changes={+ "content": [oldDocument.content, parser.get_text()],+ },+ additional_data={+ "reason": "Update document content",+ },+ action=LogEntry.Action.UPDATE,+ )- document.refresh_from_db()- logger.info(- f"Updating index for document {document_id} ({document.archive_checksum})",- )- with index.open_index_writer() as writer:- index.update_document(writer, document)+ with FileLock(settings.MEDIA_LOCK):+ if parser.get_archive_path():+ create_source_path_directory(document.archive_path)+ shutil.move(parser.get_archive_path(), document.archive_path)+ shutil.move(thumbnail, document.thumbnail_path)++ document.refresh_from_db()+ logger.info(+ f"Updating index for document {document_id} ({document.archive_checksum})",+ )+ with index.open_index_writer() as writer:+ index.update_document(writer, document)- ai_config = AIConfig()- if ai_config.llm_index_enabled:- llm_index_add_or_update_document(document)+ ai_config = AIConfig()+ if ai_config.llm_index_enabled:+ llm_index_add_or_update_document(document)- clear_document_caches(document.pk)+ clear_document_caches(document.pk)- except Exception:- logger.exception(- f"Error while parsing document {document} (ID: {document_id})",- )- finally:- # TODO(stumpylog): Remove branch in the future when all parsers use new protocol- if isinstance(- parser,- (- MailDocumentParser,- RasterisedDocumentParser,- RemoteDocumentParser,- TextDocumentParser,- TikaDocumentParser,- ),- ):- parser.__exit__(None, None, None)- else:- parser.cleanup()+ except Exception:+ logger.exception(+ f"Error while parsing document {document} (ID: {document_id})",+ )@shared_taskdiff --git a/src/documents/tests/test_checks.py b/src/documents/tests/test_checks.pyindex b78946ba9..51d9cdddc 100644--- a/src/documents/tests/test_checks.py+++ b/src/documents/tests/test_checks.py@@ -13,8 +13,10 @@ class TestDocumentChecks(TestCase):def test_parser_check(self) -> None:self.assertEqual(parser_check(None), [])- with mock.patch("documents.checks.document_consumer_declaration.send") as m:- m.return_value = []+ with mock.patch("documents.checks.get_parser_registry") as mock_registry_fn:+ mock_registry = mock.MagicMock()+ mock_registry.all_parsers.return_value = []+ mock_registry_fn.return_value = mock_registryself.assertEqual(parser_check(None),diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.pyindex a3574fdce..df4c7d9c4 100644--- a/src/documents/tests/test_consumer.py+++ b/src/documents/tests/test_consumer.py@@ -27,7 +27,6 @@ from documents.models import Documentfrom documents.models import DocumentTypefrom documents.models import StoragePathfrom documents.models import Tag-from documents.parsers import DocumentParserfrom documents.parsers import ParseErrorfrom documents.plugins.helpers import ProgressStatusOptionsfrom documents.tasks import sanity_check@@ -38,62 +37,106 @@ from documents.tests.utils import GetConsumerMixinfrom paperless_mail.models import MailRule-class _BaseTestParser(DocumentParser):- def get_settings(self) -> None:+class _BaseNewStyleParser:+ """Minimal ParserProtocol implementation for use in consumer tests."""++ name: str = "test-parser"+ version: str = "0.1"+ author: str = "test"+ url: str = "test"++ @classmethod+ def supported_mime_types(cls) -> dict:+ return {+ "application/pdf": ".pdf",+ "image/png": ".png",+ "message/rfc822": ".eml",+ }++ @classmethod+ def score(cls, mime_type: str, filename: str, path=None):+ return 0 if mime_type in cls.supported_mime_types() else None++ @property+ def can_produce_archive(self) -> bool:+ return True++ @property+ def requires_pdf_rendition(self) -> bool:+ return False++ def __init__(self) -> None:+ self._tmpdir: Path | None = None+ self._text: str | None = None+ self._archive: Path | None = None+ self._thumb: Path | None = None++ def __enter__(self):+ self._tmpdir = Path(+ tempfile.mkdtemp(prefix="paperless-test-", dir=settings.SCRATCH_DIR),+ )+ _, thumb = tempfile.mkstemp(suffix=".webp", dir=self._tmpdir)+ self._thumb = Path(thumb)+ return self++ def __exit__(self, exc_type, exc_val, exc_tb) -> None:+ if self._tmpdir and self._tmpdir.exists():+ shutil.rmtree(self._tmpdir, ignore_errors=True)++ def configure(self, context) -> None:"""- This parser does not implement additional settings yet+ Test parser doesn't do anything with context"""- return None+ def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:+ raise NotImplementedError-class DummyParser(_BaseTestParser):- def __init__(self, logging_group, scratch_dir, archive_path) -> None:- super().__init__(logging_group, None)- _, self.fake_thumb = tempfile.mkstemp(suffix=".webp", dir=scratch_dir)- self.archive_path = archive_path+ def get_text(self) -> str | None:+ return self._text- def get_thumbnail(self, document_path, mime_type, file_name=None):- return self.fake_thumb+ def get_date(self):+ return None- def parse(self, document_path, mime_type, file_name=None) -> None:- self.text = "The Text"+ def get_archive_path(self):+ return self._archive+ def get_thumbnail(self, document_path, mime_type) -> Path:+ return self._thumb-class CopyParser(_BaseTestParser):- def get_thumbnail(self, document_path, mime_type, file_name=None):- return self.fake_thumb+ def get_page_count(self, document_path, mime_type):+ return None- def __init__(self, logging_group, progress_callback=None) -> None:- super().__init__(logging_group, progress_callback)- _, self.fake_thumb = tempfile.mkstemp(suffix=".webp", dir=self.tempdir)+ def extract_metadata(self, document_path, mime_type) -> list:+ return []- def parse(self, document_path, mime_type, file_name=None) -> None:- self.text = "The text"- self.archive_path = Path(self.tempdir / "archive.pdf")- shutil.copy(document_path, self.archive_path)+class DummyParser(_BaseNewStyleParser):+ _ARCHIVE_SRC = (+ Path(__file__).parent / "samples" / "documents" / "archive" / "0000001.pdf"+ )-class FaultyParser(_BaseTestParser):- def __init__(self, logging_group, scratch_dir) -> None:- super().__init__(logging_group)- _, self.fake_thumb = tempfile.mkstemp(suffix=".webp", dir=scratch_dir)+ def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:+ self._text = "The Text"+ if produce_archive and self._tmpdir:+ self._archive = self._tmpdir / "archive.pdf"+ shutil.copy(self._ARCHIVE_SRC, self._archive)- def get_thumbnail(self, document_path, mime_type, file_name=None):- return self.fake_thumb- def parse(self, document_path, mime_type, file_name=None):- raise ParseError("Does not compute.")+class CopyParser(_BaseNewStyleParser):+ def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:+ self._text = "The text"+ if produce_archive and self._tmpdir:+ self._archive = self._tmpdir / "archive.pdf"+ shutil.copy(document_path, self._archive)-class FaultyGenericExceptionParser(_BaseTestParser):- def __init__(self, logging_group, scratch_dir) -> None:- super().__init__(logging_group)- _, self.fake_thumb = tempfile.mkstemp(suffix=".webp", dir=scratch_dir)+class FaultyParser(_BaseNewStyleParser):+ def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:+ raise ParseError("Does not compute.")- def get_thumbnail(self, document_path, mime_type, file_name=None):- return self.fake_thumb- def parse(self, document_path, mime_type, file_name=None):+class FaultyGenericExceptionParser(_BaseNewStyleParser):+ def parse(self, document_path, mime_type, *, produce_archive: bool = True) -> None:raise Exception("Generic exception.")@@ -147,38 +190,12 @@ class TestConsumer(self.assertEqual(payload["data"]["max_progress"], last_progress_max)self.assertEqual(payload["data"]["status"], last_status)- def make_dummy_parser(self, logging_group, progress_callback=None):- return DummyParser(- logging_group,- self.dirs.scratch_dir,- self.get_test_archive_file(),- )-- def make_faulty_parser(self, logging_group, progress_callback=None):- return FaultyParser(logging_group, self.dirs.scratch_dir)-- def make_faulty_generic_exception_parser(- self,- logging_group,- progress_callback=None,- ):- return FaultyGenericExceptionParser(logging_group, self.dirs.scratch_dir)-def setUp(self) -> None:super().setUp()- patcher = mock.patch("documents.parsers.document_consumer_declaration.send")- m = patcher.start()- m.return_value = [- (- None,- {- "parser": self.make_dummy_parser,- "mime_types": {"application/pdf": ".pdf"},- "weight": 0,- },- ),- ]+ patcher = mock.patch("documents.consumer.get_parser_registry")+ mock_registry = patcher.start()+ mock_registry.return_value.get_parser_for_file.return_value = DummyParserself.addCleanup(patcher.stop)def get_test_file(self):@@ -547,9 +564,9 @@ class TestConsumer() as consumer:consumer.run()- @mock.patch("documents.parsers.document_consumer_declaration.send")+ @mock.patch("documents.consumer.get_parser_registry")def testNoParsers(self, m) -> None:- m.return_value = []+ m.return_value.get_parser_for_file.return_value = Nonewith self.assertRaisesMessage(ConsumerError,@@ -560,18 +577,9 @@ class TestConsumer(self._assert_first_last_send_progress(last_status="FAILED")- @mock.patch("documents.parsers.document_consumer_declaration.send")+ @mock.patch("documents.consumer.get_parser_registry")def testFaultyParser(self, m) -> None:- m.return_value = [- (- None,- {- "parser": self.make_faulty_parser,- "mime_types": {"application/pdf": ".pdf"},- "weight": 0,- },- ),- ]+ m.return_value.get_parser_for_file.return_value = FaultyParserwith self.get_consumer(self.get_test_file()) as consumer:with self.assertRaisesMessage(@@ -582,18 +590,9 @@ class TestConsumer(self._assert_first_last_send_progress(last_status="FAILED")- @mock.patch("documents.parsers.document_consumer_declaration.send")+ @mock.patch("documents.consumer.get_parser_registry")def testGenericParserException(self, m) -> None:- m.return_value = [- (- None,- {- "parser": self.make_faulty_generic_exception_parser,- "mime_types": {"application/pdf": ".pdf"},- "weight": 0,- },- ),- ]+ m.return_value.get_parser_for_file.return_value = FaultyGenericExceptionParserwith self.get_consumer(self.get_test_file()) as consumer:with self.assertRaisesMessage(@@ -1017,7 +1016,7 @@ class TestConsumer(self._assert_first_last_send_progress()@override_settings(FILENAME_FORMAT="{title}")- @mock.patch("documents.parsers.document_consumer_declaration.send")+ @mock.patch("documents.consumer.get_parser_registry")def test_similar_filenames(self, m) -> None:shutil.copy(Path(__file__).parent / "samples" / "simple.pdf",@@ -1031,16 +1030,7 @@ class TestConsumer(Path(__file__).parent / "samples" / "simple-noalpha.png",settings.CONSUMPTION_DIR / "simple.png.pdf",)- m.return_value = [- (- None,- {- "parser": CopyParser,- "mime_types": {"application/pdf": ".pdf", "image/png": ".png"},- "weight": 0,- },- ),- ]+ m.return_value.get_parser_for_file.return_value = CopyParserwith self.get_consumer(settings.CONSUMPTION_DIR / "simple.png") as consumer:consumer.run()@@ -1068,8 +1058,10 @@ class TestConsumer(sanity_check()+ @mock.patch("documents.consumer.get_parser_registry")@mock.patch("documents.consumer.run_subprocess")- def test_try_to_clean_invalid_pdf(self, m) -> None:+ def test_try_to_clean_invalid_pdf(self, m, mock_registry) -> None:+ mock_registry.return_value.get_parser_for_file.return_value = Noneshutil.copy(Path(__file__).parent / "samples" / "invalid_pdf.pdf",settings.CONSUMPTION_DIR / "invalid_pdf.pdf",@@ -1091,10 +1083,10 @@ class TestConsumer(@mock.patch("paperless_mail.models.MailRule.objects.get")@mock.patch("paperless.parsers.mail.MailDocumentParser.parse")- @mock.patch("documents.parsers.document_consumer_declaration.send")+ @mock.patch("documents.consumer.get_parser_registry")def test_mail_parser_receives_mailrule(self,- mock_consumer_declaration_send: mock.Mock,+ mock_get_parser_registry: mock.Mock,mock_mail_parser_parse: mock.Mock,mock_mailrule_get: mock.Mock,) -> None:@@ -1106,18 +1098,11 @@ class TestConsumer(THEN:- The mail parser should receive the mail rule"""- from paperless_mail.signals import get_parser as mail_get_parser-- mock_consumer_declaration_send.return_value = [- (- None,- {- "parser": mail_get_parser,- "mime_types": {"message/rfc822": ".eml"},- "weight": 0,- },- ),- ]+ from paperless.parsers.mail import MailDocumentParser++ mock_get_parser_registry.return_value.get_parser_for_file.return_value = (+ MailDocumentParser+ )mock_mailrule_get.return_value = mock.Mock(pdf_layout=MailRule.PdfLayout.HTML_ONLY,)diff --git a/src/documents/tests/test_parsers.py b/src/documents/tests/test_parsers.pyindex 5ea1b361e..30963df70 100644--- a/src/documents/tests/test_parsers.py+++ b/src/documents/tests/test_parsers.py@@ -1,132 +1,16 @@-from tempfile import TemporaryDirectory-from unittest import mock--from django.apps import appsfrom django.test import TestCasefrom django.test import override_settingsfrom documents.parsers import get_default_file_extension-from documents.parsers import get_parser_class_for_mime_typefrom documents.parsers import get_supported_file_extensionsfrom documents.parsers import is_file_ext_supported+from paperless.parsers.registry import get_parser_registry+from paperless.parsers.registry import reset_parser_registryfrom paperless.parsers.tesseract import RasterisedDocumentParserfrom paperless.parsers.text import TextDocumentParserfrom paperless.parsers.tika import TikaDocumentParser-class TestParserDiscovery(TestCase):- @mock.patch("documents.parsers.document_consumer_declaration.send")- def test_get_parser_class_1_parser(self, m, *args) -> None:- """- GIVEN:- - Parser declared for a given mimetype- WHEN:- - Attempt to get parser for the mimetype- THEN:- - Declared parser class is returned- """-- class DummyParser:- pass-- m.return_value = (- (- None,- {- "weight": 0,- "parser": DummyParser,- "mime_types": {"application/pdf": ".pdf"},- },- ),- )-- self.assertEqual(get_parser_class_for_mime_type("application/pdf"), DummyParser)-- @mock.patch("documents.parsers.document_consumer_declaration.send")- def test_get_parser_class_n_parsers(self, m, *args) -> None:- """- GIVEN:- - Two parsers declared for a given mimetype- - Second parser has a higher weight- WHEN:- - Attempt to get parser for the mimetype- THEN:- - Second parser class is returned- """-- class DummyParser1:- pass-- class DummyParser2:- pass-- m.return_value = (- (- None,- {- "weight": 0,- "parser": DummyParser1,- "mime_types": {"application/pdf": ".pdf"},- },- ),- (- None,- {- "weight": 1,- "parser": DummyParser2,- "mime_types": {"application/pdf": ".pdf"},- },- ),- )-- self.assertEqual(- get_parser_class_for_mime_type("application/pdf"),- DummyParser2,- )-- @mock.patch("documents.parsers.document_consumer_declaration.send")- def test_get_parser_class_0_parsers(self, m, *args) -> None:- """- GIVEN:- - No parsers are declared- WHEN:- - Attempt to get parser for the mimetype- THEN:- - No parser class is returned- """- m.return_value = []- with TemporaryDirectory():- self.assertIsNone(get_parser_class_for_mime_type("application/pdf"))-- @mock.patch("documents.parsers.document_consumer_declaration.send")- def test_get_parser_class_no_valid_parser(self, m, *args) -> None:- """- GIVEN:- - No parser declared for a given mimetype- - Parser declared for a different mimetype- WHEN:- - Attempt to get parser for the given mimetype- THEN:- - No parser class is returned- """-- class DummyParser:- pass-- m.return_value = (- (- None,- {- "weight": 0,- "parser": DummyParser,- "mime_types": {"application/pdf": ".pdf"},- },- ),- )-- self.assertIsNone(get_parser_class_for_mime_type("image/tiff"))--class TestParserAvailability(TestCase):def test_tesseract_parser(self) -> None:"""@@ -151,7 +35,7 @@ class TestParserAvailability(TestCase):self.assertIn(ext, supported_exts)self.assertEqual(get_default_file_extension(mime_type), ext)self.assertIsInstance(- get_parser_class_for_mime_type(mime_type)(logging_group=None),+ get_parser_registry().get_parser_for_file(mime_type, "")(),RasterisedDocumentParser,)@@ -175,7 +59,7 @@ class TestParserAvailability(TestCase):self.assertIn(ext, supported_exts)self.assertEqual(get_default_file_extension(mime_type), ext)self.assertIsInstance(- get_parser_class_for_mime_type(mime_type)(logging_group=None),+ get_parser_registry().get_parser_for_file(mime_type, "")(),TextDocumentParser,)@@ -198,22 +82,23 @@ class TestParserAvailability(TestCase):),]- # Force the app ready to notice the settings override- with override_settings(TIKA_ENABLED=True, INSTALLED_APPS=["paperless_tika"]):- app = apps.get_app_config("paperless_tika")- app.ready()+ self.addCleanup(reset_parser_registry)++ # Reset and rebuild the registry with Tika enabled.+ with override_settings(TIKA_ENABLED=True):+ reset_parser_registry()supported_exts = get_supported_file_extensions()- for mime_type, ext in supported_mimes_and_exts:- self.assertIn(ext, supported_exts)- self.assertEqual(get_default_file_extension(mime_type), ext)- self.assertIsInstance(- get_parser_class_for_mime_type(mime_type)(logging_group=None),- TikaDocumentParser,- )+ for mime_type, ext in supported_mimes_and_exts:+ self.assertIn(ext, supported_exts)+ self.assertEqual(get_default_file_extension(mime_type), ext)+ self.assertIsInstance(+ get_parser_registry().get_parser_for_file(mime_type, "")(),+ TikaDocumentParser,+ )def test_no_parser_for_mime(self) -> None:- self.assertIsNone(get_parser_class_for_mime_type("text/sdgsdf"))+ self.assertIsNone(get_parser_registry().get_parser_for_file("text/sdgsdf", ""))def test_default_extension(self) -> None:# Test no parser declared still returns a an extensiondiff --git a/src/documents/views.py b/src/documents/views.pyindex ffdc309fd..0716ce66d 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -7,7 +7,6 @@ import tempfileimport zipfilefrom collections import defaultdictfrom collections import deque-from contextlib import nullcontextfrom datetime import datetimefrom pathlib import Pathfrom time import mktime@@ -159,7 +158,6 @@ from documents.models import UiSettingsfrom documents.models import Workflowfrom documents.models import WorkflowActionfrom documents.models import WorkflowTrigger-from documents.parsers import get_parser_class_for_mime_typefrom documents.permissions import AcknowledgeTasksPermissionsfrom documents.permissions import PaperlessAdminPermissionsfrom documents.permissions import PaperlessNotePermissions@@ -227,7 +225,7 @@ from paperless.celery import app as celery_appfrom paperless.config import AIConfigfrom paperless.config import GeneralConfigfrom paperless.models import ApplicationConfiguration-from paperless.parsers import ParserProtocol+from paperless.parsers.registry import get_parser_registryfrom paperless.serialisers import GroupSerializerfrom paperless.serialisers import UserSerializerfrom paperless.views import StandardPagination@@ -1084,17 +1082,17 @@ class DocumentViewSet(if not Path(file).is_file():return None- parser_class = get_parser_class_for_mime_type(mime_type)+ parser_class = get_parser_registry().get_parser_for_file(+ mime_type,+ Path(file).name,+ Path(file),+ )if parser_class:- parser = parser_class(progress_callback=None, logging_group=None)- cm = parser if isinstance(parser, ParserProtocol) else nullcontext(parser)-try:- with cm:+ with parser_class() as parser:return parser.extract_metadata(file, mime_type)except Exception: # pragma: no coverlogger.exception(f"Issue getting metadata for {file}")- # TODO: cover GPG errors, remove later.return []else: # pragma: no coverlogger.warning(f"No parser for {mime_type}")diff --git a/src/paperless/checks.py b/src/paperless/checks.pyindex bcea6ef24..5f069b547 100644--- a/src/paperless/checks.py+++ b/src/paperless/checks.py@@ -3,6 +3,7 @@ import osimport pwdimport shutilimport stat+import subprocessfrom pathlib import Pathfrom django.conf import settings@@ -299,3 +300,62 @@ def check_deprecated_db_settings()return warnings+++@register()+def check_remote_parser_configured(app_configs, **kwargs) -> list[Error]:+ if settings.REMOTE_OCR_ENGINE == "azureai" and not (+ settings.REMOTE_OCR_ENDPOINT and settings.REMOTE_OCR_API_KEY+ ):+ return [+ Error(+ "Azure AI remote parser requires endpoint and API key to be configured.",+ ),+ ]++ return []+++def get_tesseract_langs():+ proc = subprocess.run(+ [shutil.which("tesseract"), "--list-langs"],+ capture_output=True,+ )++ # Decode bytes to string, split on newlines, trim out the header+ proc_lines = proc.stdout.decode("utf8", errors="ignore").strip().split("\n")[1:]++ return [x.strip() for x in proc_lines]+++@register()+def check_default_language_available(app_configs, **kwargs):+ errs = []++ if not settings.OCR_LANGUAGE:+ errs.append(+ Warning(+ "No OCR language has been specified with PAPERLESS_OCR_LANGUAGE. "+ "This means that tesseract will fallback to english.",+ ),+ )+ return errs++ # binaries_check in paperless will check and report if this doesn't exist+ # So skip trying to do anything here and let that handle missing binaries+ if shutil.which("tesseract") is not None:+ installed_langs = get_tesseract_langs()++ specified_langs = [x.strip() for x in settings.OCR_LANGUAGE.split("+")]++ for lang in specified_langs:+ if lang not in installed_langs:+ errs.append(+ Error(+ f"The selected ocr language {lang} is "+ f"not installed. Paperless cannot OCR your documents "+ f"without it. Please fix PAPERLESS_OCR_LANGUAGE.",+ ),+ )++ return errsdiff --git a/src/paperless/parsers/registry.py b/src/paperless/parsers/registry.pyindex 7effe554f..c81fb1c45 100644--- a/src/paperless/parsers/registry.py+++ b/src/paperless/parsers/registry.py@@ -33,6 +33,7 @@ name, version, author, url, supported_mime_types (callable), score (callable).from __future__ import annotationsimport logging+import threadingfrom importlib.metadata import entry_pointsfrom typing import TYPE_CHECKING@@ -49,6 +50,7 @@ logger = logging.getLogger("paperless.parsers.registry")_registry: ParserRegistry | None = None_discovery_complete: bool = False+_lock = threading.Lock()# Attribute names that every registered external parser class must expose._REQUIRED_ATTRS: tuple[str, ...] = (@@ -74,7 +76,6 @@ def get_parser_registry() -> ParserRegistry:1. Creates a new ParserRegistry.2. Calls register_defaults to install built-in parsers.3. Calls discover to load third-party plugins via importlib.metadata entrypoints.- 4. Calls log_summary to emit a startup summary.Subsequent calls return the same instance immediately.@@ -85,14 +86,15 @@ def get_parser_registry() -> ParserRegistry:"""global _registry, _discovery_complete- if _registry is None:- _registry = ParserRegistry()- _registry.register_defaults()+ with _lock:+ if _registry is None:+ r = ParserRegistry()+ r.register_defaults()+ _registry = r- if not _discovery_complete:- _registry.discover()- _registry.log_summary()- _discovery_complete = True+ if not _discovery_complete:+ _registry.discover()+ _discovery_complete = Truereturn _registry@@ -113,9 +115,11 @@ def init_builtin_parsers() -> None:"""global _registry- if _registry is None:- _registry = ParserRegistry()- _registry.register_defaults()+ with _lock:+ if _registry is None:+ r = ParserRegistry()+ r.register_defaults()+ _registry = rdef reset_parser_registry() -> None:@@ -304,6 +308,23 @@ class ParserRegistry:getattr(cls, "url", "unknown"),)+ # ------------------------------------------------------------------+ # Inspection helpers+ # ------------------------------------------------------------------++ def all_parsers(self) -> list[type[ParserProtocol]]:+ """Return all registered parser classes (external first, then builtins).++ Used by compatibility wrappers that need to iterate every parser to+ compute the full set of supported MIME types and file extensions.++ Returns+ -------+ list[type[ParserProtocol]]+ External parsers followed by built-in parsers.+ """+ return [*self._external, *self._builtins]+# ------------------------------------------------------------------# Parser resolution# ------------------------------------------------------------------@@ -334,7 +355,7 @@ class ParserRegistry:mime_type:The detected MIME type of the file.filename:- The original filename, including extension.+ The original filename, including extension. May be empty in some casespath:Optional filesystem path to the file. Forwarded to eachparser's score method.diff --git a/src/paperless/settings/__init__.py b/src/paperless/settings/__init__.pyindex 011f776b5..1c33db7c6 100644--- a/src/paperless/settings/__init__.py+++ b/src/paperless/settings/__init__.py@@ -121,10 +121,7 @@ INSTALLED_APPS = ["django_extensions","paperless","documents.apps.DocumentsConfig",- "paperless_tesseract.apps.PaperlessTesseractConfig",- "paperless_text.apps.PaperlessTextConfig","paperless_mail.apps.PaperlessMailConfig",- "paperless_remote.apps.PaperlessRemoteParserConfig","django.contrib.admin","rest_framework","rest_framework.authtoken",@@ -974,8 +971,8 @@ TIKA_GOTENBERG_ENDPOINT = os.getenv("http://localhost:3000",)-if TIKA_ENABLED:- INSTALLED_APPS.append("paperless_tika.apps.PaperlessTikaConfig")+# Tika parser is now integrated into the main parser registry+# No separate Django app neededAUDIT_LOG_ENABLED = get_bool_from_env("PAPERLESS_AUDIT_LOG_ENABLED", "true")if AUDIT_LOG_ENABLED:diff --git a/src/paperless/tests/parsers/conftest.py b/src/paperless/tests/parsers/conftest.pyindex a484f02c8..8747ac9bd 100644--- a/src/paperless/tests/parsers/conftest.py+++ b/src/paperless/tests/parsers/conftest.py@@ -90,35 +90,6 @@ def text_parser() -> Generator[TextDocumentParser, None, None]:yield parser-# -------------------------------------------------------------------# Remote parser sample files-# --------------------------------------------------------------------[email protected](scope="session")-def remote_samples_dir(samples_dir: Path) -> Path:- """Absolute path to the remote parser sample files directory.-- Returns- -------- Path- ``<samples_dir>/remote/``- """- return samples_dir / "remote"--[email protected](scope="session")-def sample_pdf_file(remote_samples_dir: Path) -> Path:- """Path to a simple digital PDF sample file.-- Returns- -------- Path- Absolute path to ``remote/simple-digital.pdf``.- """- return remote_samples_dir / "simple-digital.pdf"--# ------------------------------------------------------------------# Remote parser instance# ------------------------------------------------------------------diff --git a/src/paperless/tests/parsers/test_remote_parser.py b/src/paperless/tests/parsers/test_remote_parser.pyindex 69199a6e8..892915bb5 100644--- a/src/paperless/tests/parsers/test_remote_parser.py+++ b/src/paperless/tests/parsers/test_remote_parser.py@@ -277,20 +277,20 @@ class TestRemoteParserParse:def test_parse_returns_text_from_azure(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,azure_client: Mock,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")assert remote_parser.get_text() == _DEFAULT_TEXTdef test_parse_sets_archive_path(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,azure_client: Mock,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")archive = remote_parser.get_archive_path()assert archive is not None@@ -300,11 +300,11 @@ class TestRemoteParserParse:def test_parse_closes_client_on_success(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,azure_client: Mock,) -> None:remote_parser.configure(ParserContext())- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")azure_client.close.assert_called_once()@@ -312,9 +312,9 @@ class TestRemoteParserParse:def test_parse_sets_empty_text_when_not_configured(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")assert remote_parser.get_text() == ""assert remote_parser.get_archive_path() is None@@ -328,10 +328,10 @@ class TestRemoteParserParse:def test_get_date_always_none(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,azure_client: Mock,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")assert remote_parser.get_date() is None@@ -345,33 +345,33 @@ class TestRemoteParserParseError:def test_parse_returns_none_on_azure_error(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,failing_azure_client: Mock,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")assert remote_parser.get_text() is Nonedef test_parse_closes_client_on_error(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,failing_azure_client: Mock,) -> None:- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")failing_azure_client.close.assert_called_once()def test_parse_logs_error_on_azure_failure(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,failing_azure_client: Mock,mocker: MockerFixture,) -> None:mock_log = mocker.patch("paperless.parsers.remote.logger")- remote_parser.parse(sample_pdf_file, "application/pdf")+ remote_parser.parse(simple_digital_pdf_file, "application/pdf")mock_log.error.assert_called_once()assert "Azure AI Vision parsing failed" in mock_log.error.call_args[0][0]@@ -386,18 +386,18 @@ class TestRemoteParserPageCount:def test_page_count_for_pdf(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- count = remote_parser.get_page_count(sample_pdf_file, "application/pdf")+ count = remote_parser.get_page_count(simple_digital_pdf_file, "application/pdf")assert isinstance(count, int)assert count >= 1def test_page_count_returns_none_for_image_mime(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- count = remote_parser.get_page_count(sample_pdf_file, "image/png")+ count = remote_parser.get_page_count(simple_digital_pdf_file, "image/png")assert count is Nonedef test_page_count_returns_none_for_invalid_pdf(@@ -420,25 +420,31 @@ class TestRemoteParserMetadata:def test_extract_metadata_non_pdf_returns_empty(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- result = remote_parser.extract_metadata(sample_pdf_file, "image/png")+ result = remote_parser.extract_metadata(simple_digital_pdf_file, "image/png")assert result == []def test_extract_metadata_pdf_returns_list(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- result = remote_parser.extract_metadata(sample_pdf_file, "application/pdf")+ result = remote_parser.extract_metadata(+ simple_digital_pdf_file,+ "application/pdf",+ )assert isinstance(result, list)def test_extract_metadata_pdf_entries_have_required_keys(self,remote_parser: RemoteDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- result = remote_parser.extract_metadata(sample_pdf_file, "application/pdf")+ result = remote_parser.extract_metadata(+ simple_digital_pdf_file,+ "application/pdf",+ )for entry in result:assert "namespace" in entryassert "prefix" in entrydiff --git a/src/paperless/tests/parsers/test_tika_parser.py b/src/paperless/tests/parsers/test_tika_parser.pyindex 010969259..560527934 100644--- a/src/paperless/tests/parsers/test_tika_parser.py+++ b/src/paperless/tests/parsers/test_tika_parser.py@@ -77,10 +77,10 @@ class TestTikaParserRegistryInterface:def test_get_page_count_returns_int_with_pdf_archive(self,tika_parser: TikaDocumentParser,- sample_pdf_file: Path,+ simple_digital_pdf_file: Path,) -> None:- tika_parser._archive_path = sample_pdf_file- count = tika_parser.get_page_count(sample_pdf_file, "application/pdf")+ tika_parser._archive_path = simple_digital_pdf_file+ count = tika_parser.get_page_count(simple_digital_pdf_file, "application/pdf")assert isinstance(count, int)assert count > 0diff --git a/src/paperless/tests/samples/remote/simple-digital.pdf b/src/paperless/tests/samples/remote/simple-digital.pdfdeleted file mode 100644index e450de482..000000000Binary files a/src/paperless/tests/samples/remote/simple-digital.pdf and /dev/null differdiff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.pyindex 3572f02a4..87e64a90e 100644--- a/src/paperless/tests/test_checks.py+++ b/src/paperless/tests/test_checks.py@@ -5,6 +5,7 @@ from pathlib import Pathfrom unittest import mockimport pytest+from django.core.checks import ERRORfrom django.core.checks import Errorfrom django.core.checks import Warningfrom pytest_django.fixtures import SettingsWrapper@@ -12,7 +13,9 @@ from pytest_mock import MockerFixturefrom paperless.checks import audit_log_checkfrom paperless.checks import binaries_check+from paperless.checks import check_default_language_availablefrom paperless.checks import check_deprecated_db_settings+from paperless.checks import check_remote_parser_configuredfrom paperless.checks import check_v3_minimum_upgrade_versionfrom paperless.checks import debug_mode_checkfrom paperless.checks import paths_check@@ -626,3 +629,116 @@ class TestV3MinimumUpgradeVersionCheck:conn.introspection.table_names.side_effect = OperationalError("DB unavailable")mocker.patch.dict("paperless.checks.connections", {"default": conn})assert check_v3_minimum_upgrade_version(None) == []+++class TestRemoteParserChecks:+ def test_no_engine(self, settings: SettingsWrapper) -> None:+ settings.REMOTE_OCR_ENGINE = None+ msgs = check_remote_parser_configured(None)++ assert len(msgs) == 0++ def test_azure_no_endpoint(self, settings: SettingsWrapper) -> None:++ settings.REMOTE_OCR_ENGINE = "azureai"+ settings.REMOTE_OCR_API_KEY = "somekey"+ settings.REMOTE_OCR_ENDPOINT = None++ msgs = check_remote_parser_configured(None)++ assert len(msgs) == 1++ msg = msgs[0]++ assert (+ "Azure AI remote parser requires endpoint and API key to be configured."+ in msg.msg+ )+++class TestTesseractChecks:+ def test_default_language(self) -> None:+ check_default_language_available(None)++ def test_no_language(self, settings: SettingsWrapper) -> None:++ settings.OCR_LANGUAGE = ""++ msgs = check_default_language_available(None)++ assert len(msgs) == 1+ msg = msgs[0]++ assert (+ "No OCR language has been specified with PAPERLESS_OCR_LANGUAGE" in msg.msg+ )++ def test_invalid_language(+ self,+ settings: SettingsWrapper,+ mocker: MockerFixture,+ ) -> None:++ settings.OCR_LANGUAGE = "ita"++ tesser_lang_mock = mocker.patch("paperless.checks.get_tesseract_langs")+ tesser_lang_mock.return_value = ["deu", "eng"]++ msgs = check_default_language_available(None)++ assert len(msgs) == 1+ msg = msgs[0]++ assert msg.level == ERROR+ assert "The selected ocr language ita is not installed" in msg.msg++ def test_multi_part_language(+ self,+ settings: SettingsWrapper,+ mocker: MockerFixture,+ ) -> None:+ """+ GIVEN:+ - An OCR language which is multi part (ie chi-sim)+ - The language is correctly formatted+ WHEN:+ - Installed packages are checked+ THEN:+ - No errors are reported+ """++ settings.OCR_LANGUAGE = "chi_sim"++ tesser_lang_mock = mocker.patch("paperless.checks.get_tesseract_langs")+ tesser_lang_mock.return_value = ["chi_sim", "eng"]++ msgs = check_default_language_available(None)++ assert len(msgs) == 0++ def test_multi_part_language_bad_format(+ self,+ settings: SettingsWrapper,+ mocker: MockerFixture,+ ) -> None:+ """+ GIVEN:+ - An OCR language which is multi part (ie chi-sim)+ - The language is correctly NOT formatted+ WHEN:+ - Installed packages are checked+ THEN:+ - No errors are reported+ """+ settings.OCR_LANGUAGE = "chi-sim"++ tesser_lang_mock = mocker.patch("paperless.checks.get_tesseract_langs")+ tesser_lang_mock.return_value = ["chi_sim", "eng"]++ msgs = check_default_language_available(None)++ assert len(msgs) == 1+ msg = msgs[0]++ assert msg.level == ERROR+ assert "The selected ocr language chi-sim is not installed" in msg.msgdiff --git a/src/paperless_mail/apps.py b/src/paperless_mail/apps.pyindex dd3e71f82..1c5d656e0 100644--- a/src/paperless_mail/apps.py+++ b/src/paperless_mail/apps.py@@ -1,18 +1,8 @@from django.apps import AppConfig-from django.conf import settingsfrom django.utils.translation import gettext_lazy as _-from paperless_mail.signals import mail_consumer_declaration-class PaperlessMailConfig(AppConfig):name = "paperless_mail"verbose_name = _("Paperless mail")-- def ready(self) -> None:- from documents.signals import document_consumer_declaration-- if settings.TIKA_ENABLED:- document_consumer_declaration.connect(mail_consumer_declaration)- AppConfig.ready(self)diff --git a/src/paperless_mail/signals.py b/src/paperless_mail/signals.pydeleted file mode 100644index 8fe046393..000000000--- a/src/paperless_mail/signals.py+++ /dev/null@@ -1,19 +0,0 @@-def get_parser(*args, **kwargs):- from paperless.parsers.mail import MailDocumentParser-- # MailDocumentParser accepts no constructor args in the new-style protocol.- # Pop legacy args that arrive from the signal-based consumer path.- # Phase 4 will replace this signal path with the ParserRegistry.- kwargs.pop("logging_group", None)- kwargs.pop("progress_callback", None)- return MailDocumentParser()---def mail_consumer_declaration(sender, **kwargs):- return {- "parser": get_parser,- "weight": 20,- "mime_types": {- "message/rfc822": ".eml",- },- }diff --git a/src/paperless_remote/__init__.py b/src/paperless_remote/__init__.pydeleted file mode 100644index 5380ea5ac..000000000--- a/src/paperless_remote/__init__.py+++ /dev/null@@ -1,4 +0,0 @@-# this is here so that django finds the checks.-from paperless_remote.checks import check_remote_parser_configured--__all__ = ["check_remote_parser_configured"]diff --git a/src/paperless_remote/apps.py b/src/paperless_remote/apps.pydeleted file mode 100644index 1997b0ae9..000000000--- a/src/paperless_remote/apps.py+++ /dev/null@@ -1,14 +0,0 @@-from django.apps import AppConfig--from paperless_remote.signals import remote_consumer_declaration---class PaperlessRemoteParserConfig(AppConfig):- name = "paperless_remote"-- def ready(self) -> None:- from documents.signals import document_consumer_declaration-- document_consumer_declaration.connect(remote_consumer_declaration)-- AppConfig.ready(self)diff --git a/src/paperless_remote/checks.py b/src/paperless_remote/checks.pydeleted file mode 100644index b9abb0592..000000000--- a/src/paperless_remote/checks.py+++ /dev/null@@ -1,17 +0,0 @@-from django.conf import settings-from django.core.checks import Error-from django.core.checks import register---@register()-def check_remote_parser_configured(app_configs, **kwargs):- if settings.REMOTE_OCR_ENGINE == "azureai" and not (- settings.REMOTE_OCR_ENDPOINT and settings.REMOTE_OCR_API_KEY- ):- return [- Error(- "Azure AI remote parser requires endpoint and API key to be configured.",- ),- ]-- return []diff --git a/src/paperless_remote/signals.py b/src/paperless_remote/signals.pydeleted file mode 100644index 2300be760..000000000--- a/src/paperless_remote/signals.py+++ /dev/null@@ -1,38 +0,0 @@-from __future__ import annotations--from typing import Any---def get_parser(*args: Any, **kwargs: Any) -> Any:- from paperless.parsers.remote import RemoteDocumentParser-- # The new RemoteDocumentParser does not accept the progress_callback- # kwarg injected by the old signal-based consumer. logging_group is- # forwarded as a positional arg.- # Phase 4 will replace this signal path with the new ParserRegistry.- kwargs.pop("progress_callback", None)- return RemoteDocumentParser(*args, **kwargs)---def get_supported_mime_types() -> dict[str, str]:- from django.conf import settings-- from paperless.parsers.remote import RemoteDocumentParser- from paperless.parsers.remote import RemoteEngineConfig-- config = RemoteEngineConfig(- engine=settings.REMOTE_OCR_ENGINE,- api_key=settings.REMOTE_OCR_API_KEY,- endpoint=settings.REMOTE_OCR_ENDPOINT,- )- if not config.engine_is_valid():- return {}- return RemoteDocumentParser.supported_mime_types()---def remote_consumer_declaration(sender: Any, **kwargs: Any) -> dict[str, Any]:- return {- "parser": get_parser,- "weight": 5,- "mime_types": get_supported_mime_types(),- }diff --git a/src/paperless_remote/tests/__init__.py b/src/paperless_remote/tests/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/paperless_remote/tests/test_checks.py b/src/paperless_remote/tests/test_checks.pydeleted file mode 100644index 0512fb257..000000000--- a/src/paperless_remote/tests/test_checks.py+++ /dev/null@@ -1,24 +0,0 @@-from unittest import TestCase--from django.test import override_settings--from paperless_remote import check_remote_parser_configured---class TestChecks(TestCase):- @override_settings(REMOTE_OCR_ENGINE=None)- def test_no_engine(self) -> None:- msgs = check_remote_parser_configured(None)- self.assertEqual(len(msgs), 0)-- @override_settings(REMOTE_OCR_ENGINE="azureai")- @override_settings(REMOTE_OCR_API_KEY="somekey")- @override_settings(REMOTE_OCR_ENDPOINT=None)- def test_azure_no_endpoint(self) -> None:- msgs = check_remote_parser_configured(None)- self.assertEqual(len(msgs), 1)- self.assertTrue(- msgs[0].msg.startswith(- "Azure AI remote parser requires endpoint and API key to be configured.",- ),- )diff --git a/src/paperless_tesseract/__init__.py b/src/paperless_tesseract/__init__.pydeleted file mode 100644index cc0b886aa..000000000--- a/src/paperless_tesseract/__init__.py+++ /dev/null@@ -1,5 +0,0 @@-# this is here so that django finds the checks.-from paperless_tesseract.checks import check_default_language_available-from paperless_tesseract.checks import get_tesseract_langs--__all__ = ["check_default_language_available", "get_tesseract_langs"]diff --git a/src/paperless_tesseract/apps.py b/src/paperless_tesseract/apps.pydeleted file mode 100644index 8ade88400..000000000--- a/src/paperless_tesseract/apps.py+++ /dev/null@@ -1,14 +0,0 @@-from django.apps import AppConfig--from paperless_tesseract.signals import tesseract_consumer_declaration---class PaperlessTesseractConfig(AppConfig):- name = "paperless_tesseract"-- def ready(self) -> None:- from documents.signals import document_consumer_declaration-- document_consumer_declaration.connect(tesseract_consumer_declaration)-- AppConfig.ready(self)diff --git a/src/paperless_tesseract/checks.py b/src/paperless_tesseract/checks.pydeleted file mode 100644index 0d7a1d90d..000000000--- a/src/paperless_tesseract/checks.py+++ /dev/null@@ -1,52 +0,0 @@-import shutil-import subprocess--from django.conf import settings-from django.core.checks import Error-from django.core.checks import Warning-from django.core.checks import register---def get_tesseract_langs():- proc = subprocess.run(- [shutil.which("tesseract"), "--list-langs"],- capture_output=True,- )-- # Decode bytes to string, split on newlines, trim out the header- proc_lines = proc.stdout.decode("utf8", errors="ignore").strip().split("\n")[1:]-- return [x.strip() for x in proc_lines]---@register()-def check_default_language_available(app_configs, **kwargs):- errs = []-- if not settings.OCR_LANGUAGE:- errs.append(- Warning(- "No OCR language has been specified with PAPERLESS_OCR_LANGUAGE. "- "This means that tesseract will fallback to english.",- ),- )- return errs-- # binaries_check in paperless will check and report if this doesn't exist- # So skip trying to do anything here and let that handle missing binaries- if shutil.which("tesseract") is not None:- installed_langs = get_tesseract_langs()-- specified_langs = [x.strip() for x in settings.OCR_LANGUAGE.split("+")]-- for lang in specified_langs:- if lang not in installed_langs:- errs.append(- Error(- f"The selected ocr language {lang} is "- f"not installed. Paperless cannot OCR your documents "- f"without it. Please fix PAPERLESS_OCR_LANGUAGE.",- ),- )-- return errsdiff --git a/src/paperless_tesseract/signals.py b/src/paperless_tesseract/signals.pydeleted file mode 100644index d80d13614..000000000--- a/src/paperless_tesseract/signals.py+++ /dev/null@@ -1,34 +0,0 @@-from __future__ import annotations--from typing import Any---def get_parser(*args: Any, **kwargs: Any) -> Any:- from paperless.parsers.tesseract import RasterisedDocumentParser-- # RasterisedDocumentParser accepts logging_group for constructor compatibility but- # does not store or use it (no legacy DocumentParser base class).- # progress_callback is also not used. Both may arrive as a positional arg- # (consumer) or a keyword arg (views); *args absorbs the positional form,- # kwargs.pop handles the keyword form. Phase 4 will replace this signal- # path with the new ParserRegistry so the shim can be removed at that point.- kwargs.pop("logging_group", None)- kwargs.pop("progress_callback", None)- return RasterisedDocumentParser(*args, **kwargs)---def tesseract_consumer_declaration(sender: Any, **kwargs: Any) -> dict[str, Any]:- return {- "parser": get_parser,- "weight": 0,- "mime_types": {- "application/pdf": ".pdf",- "image/jpeg": ".jpg",- "image/png": ".png",- "image/tiff": ".tif",- "image/gif": ".gif",- "image/bmp": ".bmp",- "image/webp": ".webp",- "image/heic": ".heic",- },- }diff --git a/src/paperless_tesseract/tests/__init__.py b/src/paperless_tesseract/tests/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/paperless_tesseract/tests/test_checks.py b/src/paperless_tesseract/tests/test_checks.pydeleted file mode 100644index ab3ba0c16..000000000--- a/src/paperless_tesseract/tests/test_checks.py+++ /dev/null@@ -1,67 +0,0 @@-from unittest import mock--from django.core.checks import ERROR-from django.test import TestCase-from django.test import override_settings--from paperless_tesseract import check_default_language_available---class TestChecks(TestCase):- def test_default_language(self) -> None:- check_default_language_available(None)-- @override_settings(OCR_LANGUAGE="")- def test_no_language(self) -> None:- msgs = check_default_language_available(None)- self.assertEqual(len(msgs), 1)- self.assertTrue(- msgs[0].msg.startswith(- "No OCR language has been specified with PAPERLESS_OCR_LANGUAGE",- ),- )-- @override_settings(OCR_LANGUAGE="ita")- @mock.patch("paperless_tesseract.checks.get_tesseract_langs")- def test_invalid_language(self, m) -> None:- m.return_value = ["deu", "eng"]- msgs = check_default_language_available(None)- self.assertEqual(len(msgs), 1)- self.assertEqual(msgs[0].level, ERROR)-- @override_settings(OCR_LANGUAGE="chi_sim")- @mock.patch("paperless_tesseract.checks.get_tesseract_langs")- def test_multi_part_language(self, m) -> None:- """- GIVEN:- - An OCR language which is multi part (ie chi-sim)- - The language is correctly formatted- WHEN:- - Installed packages are checked- THEN:- - No errors are reported- """- m.return_value = ["chi_sim", "eng"]-- msgs = check_default_language_available(None)-- self.assertEqual(len(msgs), 0)-- @override_settings(OCR_LANGUAGE="chi-sim")- @mock.patch("paperless_tesseract.checks.get_tesseract_langs")- def test_multi_part_language_bad_format(self, m) -> None:- """- GIVEN:- - An OCR language which is multi part (ie chi-sim)- - The language is correctly NOT formatted- WHEN:- - Installed packages are checked- THEN:- - No errors are reported- """- m.return_value = ["chi_sim", "eng"]-- msgs = check_default_language_available(None)-- self.assertEqual(len(msgs), 1)- self.assertEqual(msgs[0].level, ERROR)diff --git a/src/paperless_text/__init__.py b/src/paperless_text/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/paperless_text/apps.py b/src/paperless_text/apps.pydeleted file mode 100644index 619d71886..000000000--- a/src/paperless_text/apps.py+++ /dev/null@@ -1,14 +0,0 @@-from django.apps import AppConfig--from paperless_text.signals import text_consumer_declaration---class PaperlessTextConfig(AppConfig):- name = "paperless_text"-- def ready(self) -> None:- from documents.signals import document_consumer_declaration-- document_consumer_declaration.connect(text_consumer_declaration)-- AppConfig.ready(self)diff --git a/src/paperless_text/signals.py b/src/paperless_text/signals.pydeleted file mode 100644index 916f0a7c0..000000000--- a/src/paperless_text/signals.py+++ /dev/null@@ -1,29 +0,0 @@-from __future__ import annotations--from typing import Any---def get_parser(*args: Any, **kwargs: Any) -> Any:- from paperless.parsers.text import TextDocumentParser-- # TextDocumentParser accepts logging_group for constructor compatibility but- # does not store or use it (no legacy DocumentParser base class).- # progress_callback is also not used. Both may arrive as a positional arg- # (consumer) or a keyword arg (views); *args absorbs the positional form,- # kwargs.pop handles the keyword form. Phase 4 will replace this signal- # path with the new ParserRegistry so the shim can be removed at that point.- kwargs.pop("logging_group", None)- kwargs.pop("progress_callback", None)- return TextDocumentParser(*args, **kwargs)---def text_consumer_declaration(sender: Any, **kwargs: Any) -> dict[str, Any]:- return {- "parser": get_parser,- "weight": 10,- "mime_types": {- "text/plain": ".txt",- "text/csv": ".csv",- "application/csv": ".csv",- },- }diff --git a/src/paperless_text/tests/__init__.py b/src/paperless_text/tests/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/paperless_tika/__init__.py b/src/paperless_tika/__init__.pydeleted file mode 100644index e69de29bb..000000000diff --git a/src/paperless_tika/apps.py b/src/paperless_tika/apps.pydeleted file mode 100644index 714a05188..000000000--- a/src/paperless_tika/apps.py+++ /dev/null@@ -1,15 +0,0 @@-from django.apps import AppConfig-from django.conf import settings--from paperless_tika.signals import tika_consumer_declaration---class PaperlessTikaConfig(AppConfig):- name = "paperless_tika"-- def ready(self) -> None:- from documents.signals import document_consumer_declaration-- if settings.TIKA_ENABLED:- document_consumer_declaration.connect(tika_consumer_declaration)- AppConfig.ready(self)diff --git a/src/paperless_tika/signals.py b/src/paperless_tika/signals.pydeleted file mode 100644index f1fd17ef6..000000000--- a/src/paperless_tika/signals.py+++ /dev/null@@ -1,33 +0,0 @@-def get_parser(*args, **kwargs):- from paperless.parsers.tika import TikaDocumentParser-- # TikaDocumentParser accepts logging_group for constructor compatibility but- # does not store or use it (no legacy DocumentParser base class).- # progress_callback is also not used. Both may arrive as a positional arg- # (consumer) or a keyword arg (views); *args absorbs the positional form,- # kwargs.pop handles the keyword form. Phase 4 will replace this signal- # path with the new ParserRegistry so the shim can be removed at that point.- kwargs.pop("logging_group", None)- kwargs.pop("progress_callback", None)- return TikaDocumentParser()---def tika_consumer_declaration(sender, **kwargs):- return {- "parser": get_parser,- "weight": 10,- "mime_types": {- "application/msword": ".doc",- "application/vnd.openxmlformats-officedocument.wordprocessingml.document": ".docx",- "application/vnd.ms-excel": ".xls",- "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet": ".xlsx",- "application/vnd.ms-powerpoint": ".ppt",- "application/vnd.openxmlformats-officedocument.presentationml.presentation": ".pptx",- "application/vnd.openxmlformats-officedocument.presentationml.slideshow": ".ppsx",- "application/vnd.oasis.opendocument.presentation": ".odp",- "application/vnd.oasis.opendocument.spreadsheet": ".ods",- "application/vnd.oasis.opendocument.text": ".odt",- "application/vnd.oasis.opendocument.graphics": ".odg",- "text/rtf": ".rtf",- },- }