A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Performance: resolve index-write permissions and effective content in bulk (#13869)
Trenton Hf5ff18326d2d
diff --git a/src/documents/search/_backend.py b/src/documents/search/_backend.pyindex b783cfd2d..e5bae979e 100644--- a/src/documents/search/_backend.py+++ b/src/documents/search/_backend.py@@ -284,6 +284,46 @@ class WriteBatch:tantivy.Query.term_query(self._backend._schema, "id", doc_id),)+ def add_or_update_ids(self, ids: Sequence[int]) -> None:+ """+ Add or update multiple documents in the batch by primary key.++ Unlike calling ``add_or_update()`` once per document, this resolves+ viewer permissions and effective (versioned) content in bulk against+ the ids as a whole, instead of once per document -- see+ ``_DocumentViewerStream`` and ``annotate_effective_content``. Use+ this whenever more than one document is being written in the same+ batch.++ An id with no matching document (e.g. deleted between the caller+ collecting ids and the batch running) is silently skipped, matching+ ``add_or_update()``'s existing single-document deferred-task behavior+ rather than erroring or leaving a stale index entry.++ Args:+ ids: Primary keys of Document instances to index+ """+ from documents.models import Document+ from documents.versioning import annotate_effective_content++ ids = list(ids)+ if not ids:+ return++ queryset = annotate_effective_content(+ Document.objects.filter(pk__in=ids)+ .select_related("correspondent", "document_type", "storage_path", "owner")+ .prefetch_related("tags", "notes__user", "custom_fields__field"),+ )+ for document, grant in _DocumentViewerStream(queryset, chunk_size=1000):+ self.remove(document.pk)+ doc = self._backend._build_tantivy_doc(+ document,+ viewer_ids=grant.viewer_ids,+ viewer_group_ids=grant.viewer_group_ids,+ )+ self._writer.add_document(doc)+class TantivyBackend:"""diff --git a/src/documents/tasks.py b/src/documents/tasks.pyindex ca53468eb..33f7d4f90 100644--- a/src/documents/tasks.py+++ b/src/documents/tasks.py@@ -312,7 +312,10 @@ def bulk_update_documents(document_ids) -> None:from documents.search import get_backenddocument_ids = list(document_ids)- # Annotated so indexing below doesn't query the versions of each document+ # Annotated so the signal handlers below (e.g. matching) don't query the+ # versions of each document. Indexing re-queries and re-annotates its own+ # copy via add_or_update_ids() below, after these signals (and any+ # workflow they trigger) have had a chance to mutate the documents.documents = annotate_effective_content(Document.objects.filter(id__in=document_ids),)@@ -328,8 +331,7 @@ def bulk_update_documents(document_ids) -> None:post_save.send(Document, instance=doc, created=False)with get_backend().batch_update() as batch:- for doc in documents:- batch.add_or_update(doc)+ batch.add_or_update_ids(document_ids)ai_config = AIConfig()if ai_config.llm_index_enabled:diff --git a/src/documents/tests/search/test_backend.py b/src/documents/tests/search/test_backend.pyindex 21fed8f02..60b5d62da 100644--- a/src/documents/tests/search/test_backend.py+++ b/src/documents/tests/search/test_backend.py@@ -4,6 +4,8 @@ from pathlib import Pathimport pytestfrom django.contrib.auth.models import Groupfrom django.contrib.auth.models import User+from django.db import connection+from django.test.utils import CaptureQueriesContextfrom guardian.shortcuts import assign_permfrom pytest_mock import MockerFixture@@ -102,6 +104,191 @@ class TestWriteBatch:assert len(backend.search_ids("indexable", user=None)) == 1+class TestAddOrUpdateIds:+ """Test WriteBatch.add_or_update_ids(), the bulk id-based upsert path.++ Unlike add_or_update() called once per document, this resolves viewer+ permissions and effective (versioned) content in bulk against the ids as+ a whole, so it must produce identical indexed output to the per-document+ path while issuing a constant number of queries regardless of batch size.+ """++ def test_missing_id_is_skipped_not_errored(+ self,+ backend: TantivyBackend,+ ) -> None:+ doc = Document.objects.create(+ title="doc",+ content="present",+ checksum="EXIST1",+ pk=1,+ )+ missing_pk = 999++ with backend.batch_update() as batch:+ batch.add_or_update_ids([doc.pk, missing_pk])++ assert backend.search_ids("present", user=None) == [doc.pk]++ def test_query_count_does_not_scale_with_batch_size(+ self,+ backend: TantivyBackend,+ ) -> None:+ """Each query count must stay far below N, not merely match between+ two runs -- an exact-equality assertion between two measurements is+ at the mercy of incidental process-level caches (e.g. Django's+ ContentType.objects.get_for_model) warming on whichever run happens+ first, which makes counts differ by a query for reasons unrelated to+ batch size. A generous fixed bound sidesteps that: the old+ per-document path issued roughly 8 queries per document, so 50+ documents under a bound this low proves the fix regardless of cache+ state.+ """+ max_queries_for_any_batch_size = 15++ small_docs = [+ Document.objects.create(+ title="doc",+ content=f"unique{i}",+ checksum=f"SMALL{i}",+ pk=i,+ )+ for i in range(1, 3)+ ]+ with CaptureQueriesContext(connection) as ctx_small:+ with backend.batch_update() as batch:+ batch.add_or_update_ids([d.pk for d in small_docs])+ assert len(ctx_small.captured_queries) <= max_queries_for_any_batch_size++ large_docs = [+ Document.objects.create(+ title="doc",+ content=f"unique{i}",+ checksum=f"LARGE{i}",+ pk=i,+ )+ for i in range(100, 150)+ ]+ with CaptureQueriesContext(connection) as ctx_large:+ with backend.batch_update() as batch:+ batch.add_or_update_ids([d.pk for d in large_docs])+ assert len(ctx_large.captured_queries) <= max_queries_for_any_batch_size++ for doc in large_docs:+ assert backend.search_ids(f"unique{doc.pk}", user=None) == [doc.pk]++ def test_resolves_direct_user_grant_in_bulk(+ self,+ backend: TantivyBackend,+ ) -> None:+ owner = UserFactory()+ user = UserFactory()+ doc = Document.objects.create(+ title="doc",+ checksum="PERM1",+ pk=1,+ owner=owner,+ )+ assign_perm("view_document", user, doc)++ with backend.batch_update() as batch:+ batch.add_or_update_ids([doc.pk])++ assert backend.search_ids("doc", user=user) == [doc.pk]+ other = UserFactory()+ assert backend.search_ids("doc", user=other) == []++ def test_resolves_group_grant_in_bulk(self, backend: TantivyBackend) -> None:+ owner = UserFactory()+ group = Group.objects.create(name="reviewers")+ user = UserFactory()+ user.groups.add(group)+ doc = Document.objects.create(+ title="doc",+ checksum="GPERM1",+ pk=1,+ owner=owner,+ )+ assign_perm("view_document", group, doc)++ with backend.batch_update() as batch:+ batch.add_or_update_ids([doc.pk])++ assert backend.search_ids("doc", user=user) == [doc.pk]+ other = UserFactory()+ assert backend.search_ids("doc", user=other) == []++ def test_indexes_notes_and_custom_fields(self, backend: TantivyBackend) -> None:+ note_author = UserFactory(username="noter")+ field = CustomField.objects.create(+ name="Invoice Number",+ data_type=CustomField.FieldDataType.STRING,+ )+ doc = Document.objects.create(title="doc", checksum="RICH1", pk=1)+ Note.objects.create(document=doc, note="Reviewed", user=note_author)+ CustomFieldInstance.objects.create(+ document=doc,+ field=field,+ value_text="INV-42",+ )++ with backend.batch_update() as batch:+ batch.add_or_update_ids([doc.pk])++ assert backend.search_ids("notes.user:noter", user=None) == [doc.pk]+ assert backend.search_ids("custom_fields.value:INV-42", user=None) == [+ doc.pk,+ ]++ def test_uses_effective_content_for_versioned_documents(+ self,+ backend: TantivyBackend,+ ) -> None:+ root = Document.objects.create(+ title="Statement",+ content="stale text",+ checksum="ROOT1",+ pk=1,+ )+ Document.objects.create(+ title="Statement",+ content="latest version text",+ checksum="VER1",+ pk=2,+ root_document=root,+ version_index=1,+ )++ with backend.batch_update() as batch:+ batch.add_or_update_ids([root.pk])++ assert backend.search_ids("latest", user=None) == [root.pk]+ assert backend.search_ids("stale", user=None) == []++ def test_reindexes_documents_already_in_the_index(+ self,+ backend: TantivyBackend,+ ) -> None:+ """add_or_update_ids must upsert, matching add_or_update's behaviour."""+ doc = Document.objects.create(+ title="doc",+ content="original",+ checksum="UP1",+ pk=1,+ )+ backend.add_or_update(doc)+ assert backend.search_ids("original", user=None) == [doc.pk]++ doc.content = "updated"+ doc.save()++ with backend.batch_update() as batch:+ batch.add_or_update_ids([doc.pk])++ assert backend.search_ids("original", user=None) == []+ assert backend.search_ids("updated", user=None) == [doc.pk]++class TestSearch:"""Test search query parsing and matching via search_ids."""diff --git a/src/documents/views.py b/src/documents/views.pyindex 0d256115a..4cf775bd6 100644--- a/src/documents/views.py+++ b/src/documents/views.py@@ -5469,8 +5469,7 @@ class TrashView(ListModelMixin, PassUserMixin):from documents.search import get_backendwith get_backend().batch_update() as batch:- for doc in restored:- batch.add_or_update(doc)+ batch.add_or_update_ids([doc.pk for doc in restored])elif action == "empty":if doc_ids is None:doc_ids = [doc.id for doc in docs]