A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Chore: Delete the in-memory Tantivy backend and run the search tests on disk (#14216)
Trenton He4367b5648a7
diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.pyindex 447bc6bbe..c04ccbfd4 100644--- a/src/documents/search/_backend.py+++ b/src/documents/search/_backend.py@@ -196,52 +196,49 @@ class WriteBatch:return self._raw_writerdef __enter__(self) -> Self:- if self._backend._path is not None:- lock_path = self._backend._path / ".tantivy.lock"- self._lock = filelock.FileLock(str(lock_path))- for attempt in range(_LOCK_RETRY_ATTEMPTS):- try:- self._lock.acquire(timeout=self._lock_timeout)- break- except filelock.Timeout:- if attempt == _LOCK_RETRY_ATTEMPTS - 1:- raise SearchIndexLockError(- f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "- f"attempts (timeout={self._lock_timeout}s each)",- )- sleep_s = random.uniform(- 0,- min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),- )- logger.debug(- "Index lock contention; retrying in %.2fs (attempt %d/%d)",- sleep_s,- attempt + 1,- _LOCK_RETRY_ATTEMPTS,+ lock_path = self._backend._path / ".tantivy.lock"+ self._lock = filelock.FileLock(str(lock_path))+ for attempt in range(_LOCK_RETRY_ATTEMPTS):+ try:+ self._lock.acquire(timeout=self._lock_timeout)+ break+ except filelock.Timeout:+ if attempt == _LOCK_RETRY_ATTEMPTS - 1:+ raise SearchIndexLockError(+ f"Could not acquire index lock after {_LOCK_RETRY_ATTEMPTS} "+ f"attempts (timeout={self._lock_timeout}s each)",)- time.sleep(sleep_s)-- # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)- # for the write, rather than reusing the process-local cached- # index. ManagedDirectory loads its GC bookkeeping (.managed.json)- # once, at construction, and never re-reads it; paperless runs- # several long-lived processes (Granian workers, Celery workers)- # that take turns writing under the file lock above. A cached,- # long-lived writer index would carry a stale managed-files view- # and, on commit, overwrite .managed.json with that stale view -- # permanently losing track of segment files other processes- # registered in the meantime, so they can never be garbage- # collected. Reopening fresh here always picks up the current- # on-disk state. The long-lived self._backend._index is used for- # reads only and is reloaded (not reopened) after commit below.- write_index = tantivy.Index(- build_schema(),- path=str(self._backend._path),- )- register_tokenizers(write_index, settings.SEARCH_LANGUAGE)- self._raw_writer = write_index.writer()- else:- self._raw_writer = self._backend._index.writer()+ sleep_s = random.uniform(+ 0,+ min(_LOCK_BACKOFF_CAP, _LOCK_BACKOFF_BASE * (2**attempt)),+ )+ logger.debug(+ "Index lock contention; retrying in %.2fs (attempt %d/%d)",+ sleep_s,+ attempt + 1,+ _LOCK_RETRY_ATTEMPTS,+ )+ time.sleep(sleep_s)++ # Open a fresh Index (and thus a fresh Tantivy ManagedDirectory)+ # for the write, rather than reusing the process-local cached+ # index. ManagedDirectory loads its GC bookkeeping (.managed.json)+ # once, at construction, and never re-reads it; paperless runs+ # several long-lived processes (Granian workers, Celery workers)+ # that take turns writing under the file lock above. A cached,+ # long-lived writer index would carry a stale managed-files view+ # and, on commit, overwrite .managed.json with that stale view -+ # permanently losing track of segment files other processes+ # registered in the meantime, so they can never be garbage+ # collected. Reopening fresh here always picks up the current+ # on-disk state. The long-lived self._backend._index is used for+ # reads only and is reloaded (not reopened) after commit below.+ write_index = tantivy.Index(+ build_schema(),+ path=str(self._backend._path),+ )+ register_tokenizers(write_index, settings.SEARCH_LANGUAGE)+ self._raw_writer = write_index.writer()return selfdef __exit__(self, exc_type, exc_val, exc_tb):@@ -372,9 +369,8 @@ class TantivyBackend:Tantivy search backend with explicit lifecycle management.Provides full-text search capabilities using the Tantivy search engine.- Supports in-memory indexes (for testing) and persistent on-disk indexes- (for production use). Handles document indexing, search queries, autocompletion,- and "more like this" functionality.+ Keeps a persistent on-disk index. Handles document indexing, search queries,+ autocompletion, and "more like this" functionality.The backend manages its own connection lifecycle and can be reset whenthe underlying index directory changes (e.g., during test isolation).@@ -408,9 +404,7 @@ class TantivyBackend:},)- def __init__(self, path: Path | None = None):- # path=None → in-memory index (for tests)- # path=some_dir → on-disk index (for production)+ def __init__(self, path: Path):self._path = pathself._raw_index: tantivy.Index | None = Noneself._raw_schema: tantivy.Schema | None = None@@ -429,16 +423,13 @@ class TantivyBackend:"""Open or rebuild the index as needed.- For disk-based indexes, checks if rebuilding is needed due to schema- version or language changes. Registers custom tokenizers after opening.+ Checks if rebuilding is needed due to schema version or language+ changes. Registers custom tokenizers after opening.Safe to call multiple times - subsequent calls are no-ops."""if self._raw_index is not None:return # pragma: no cover- if self._path is not None:- self._raw_index = open_or_rebuild_index(self._path)- else:- self._raw_index = tantivy.Index(build_schema())+ self._raw_index = open_or_rebuild_index(self._path)register_tokenizers(self._raw_index, settings.SEARCH_LANGUAGE)self._raw_schema = self._raw_index.schema@@ -1102,13 +1093,9 @@ class TantivyBackend:writer's threads). Larger values buffer more docs in RAM beforeflushing a segment, deferring merge work; they do not avoid it."""- # Create new index (on-disk or in-memory)- if self._path is not None:- wipe_index(self._path)- new_index = tantivy.Index(build_schema(), path=str(self._path))- _write_sentinels(self._path)- else:- new_index = tantivy.Index(build_schema())+ wipe_index(self._path)+ new_index = tantivy.Index(build_schema(), path=str(self._path))+ _write_sentinels(self._path)register_tokenizers(new_index, settings.SEARCH_LANGUAGE)# Point instance at the new index so _build_tantivy_doc uses itdiff --git a/src/documents/tests/search/conftest.py b/src/documents/tests/search/conftest.pyindex ce805cdd4..abeb99d76 100644--- a/src/documents/tests/search/conftest.py+++ b/src/documents/tests/search/conftest.py@@ -19,6 +19,7 @@ if TYPE_CHECKING:from pytest_django.fixtures import Settingsfrom documents.models import Document+ from paperless_testing.dirs import PaperlessDirs@pytest.fixture@@ -30,8 +31,8 @@ def index_dir(tmp_path: Path, settings: Settings) -> Path:@pytest.fixture-def backend() -> Generator[TantivyBackend, None, None]:- b = TantivyBackend() # path=None → in-memory index+def backend(paperless_dirs: PaperlessDirs) -> Generator[TantivyBackend, None, None]:+ b = TantivyBackend(path=paperless_dirs.index_dir)b.open()try:yield bdiff --git a/src/paperless_ai/tests/test_ai_classifier.py b/src/paperless_ai/tests/test_ai_classifier.pyindex cbdd1e7f1..eeb8eabf7 100644--- a/src/paperless_ai/tests/test_ai_classifier.py+++ b/src/paperless_ai/tests/test_ai_classifier.py@@ -1,6 +1,7 @@import datetimefrom collections.abc import Generatorfrom types import SimpleNamespace+from typing import TYPE_CHECKINGfrom unittest.mock import MagicMockfrom unittest.mock import patch@@ -28,6 +29,9 @@ from paperless_testing.factories import TagFactoryfrom paperless_testing.factories import UserFactoryfrom paperless_testing.permissions import grant_object+if TYPE_CHECKING:+ from paperless_testing.dirs import PaperlessDirs+@pytest.fixturedef mock_document():@@ -630,10 +634,11 @@ class TestFulltextSimilarDocuments:def fulltext_backend(self,mocker: pytest_mock.MockerFixture,+ paperless_dirs: "PaperlessDirs",) -> Generator[TantivyBackend, None, None]:- """An in-memory Tantivy backend, wired up as the module-level+ """An on-disk Tantivy backend, wired up as the module-levelsingleton _fulltext_similar_documents resolves via get_backend()."""- backend = TantivyBackend(path=None)+ backend = TantivyBackend(path=paperless_dirs.index_dir)backend.open()mocker.patch("documents.search.get_backend", return_value=backend)try: