A community-supported supercharged document management system: scan, index and archive all your documents
Python aiangulararchivingdjangodmsdocument-managementdocument-management-systemllmmachine-learningocroptical-character-recognitionpdf
Chore: Move the search tests onto the shared indexing fixtures (#14159)
Trenton Hfdd32b73bbcc
diff --git a/src/documents/tests/conftest.py b/src/documents/tests/conftest.pyindex 99f53d614..626760d31 100644--- a/src/documents/tests/conftest.py+++ b/src/documents/tests/conftest.py@@ -182,3 +182,15 @@ def faker_session_locale():@pytest.fixture(scope="session", autouse=True)def faker_seed():return 12345+++def indexed_document(_search_index: None) -> "Document":+ """One searchable document, for tests about what the search endpoint+ returns rather than about what it finds.+ """+ from documents.search import get_backend++ doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")+ get_backend().add_or_update(doc)+ return docdiff --git a/src/documents/tests/search/_ast_helpers.py b/src/documents/tests/search/_ast_helpers.pynew file mode 100644index 000000000..aa38c25af--- /dev/null+++ b/src/documents/tests/search/_ast_helpers.py@@ -0,0 +1,30 @@+"""Field references and leaf constructors shared by the unit-level tests.++These are pure whoosh-compat AST values with no state and no database+behind them, so they stay plain module-level functions rather than+fixtures. ``NOTES`` is the reason this module exists at all: notes is a+JSON field, so addressing its text means naming a subpath, and spelling+``wc.FieldRef("notes", "note")`` out once per test file invites two of+them to disagree.+"""++from __future__ import annotations++import whoosh_compat as wc+import whoosh_compat.ast as wc_ast++CONTENT = wc.FieldRef("content")+TITLE = wc.FieldRef("title")+NOTES = wc.FieldRef("notes", "note")+BIGRAM_CONTENT = wc.FieldRef("bigram_content")+BIGRAM_TITLE = wc.FieldRef("bigram_title")+++def content(text: str) -> wc_ast.Term:+ """A Term on the content field, the default leaf these tests widen."""+ return wc_ast.Term(field=CONTENT, text=text)+++def bigram(text: str) -> wc_ast.Term:+ """A Term on the bigram side of content."""+ return wc_ast.Term(field=BIGRAM_CONTENT, text=text)diff --git a/src/documents/tests/search/conftest.py b/src/documents/tests/search/conftest.pyindex 7bb065cb0..64dfcfe0c 100644--- a/src/documents/tests/search/conftest.py+++ b/src/documents/tests/search/conftest.py@@ -3,9 +3,12 @@ from __future__ import annotationsfrom typing import TYPE_CHECKINGimport pytest+import tantivyfrom documents.search._backend import TantivyBackendfrom documents.search._backend import reset_backend+from documents.search._schema import build_schema+from documents.search._tokenizer import register_tokenizersfrom documents.tests.factories import DocumentFactoryif TYPE_CHECKING:@@ -72,3 +75,23 @@ def matched_ids(backend: TantivyBackend) -> Callable[[str], set[int]]:return set(backend.search_ids(query, user=None))return _matched_ids++[email protected](scope="module")+def query_index() -> tantivy.Index:+ """An in-memory, unstemmed index for the parse-only tests.++ These never index a document, so one index per module is shared+ read-only across that module's tests.+ """+ idx = tantivy.Index(build_schema(), path=None)+ register_tokenizers(idx, "")+ return idx+++def fuzzy_enabled(settings: Settings) -> None:+ """Enable the fuzzy blend clause. The threshold doubles as a minimum+ score filter, so it is set to 0.0: every hit passes and the test sees+ the clause's matching behaviour, not the filter's."""+ settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0diff --git a/src/documents/tests/search/test_acceptance.py b/src/documents/tests/search/test_acceptance.pyindex 10fd41fc8..097fc28de 100644--- a/src/documents/tests/search/test_acceptance.py+++ b/src/documents/tests/search/test_acceptance.py@@ -18,13 +18,16 @@ from django.contrib.auth.models import Userfrom documents.models import CustomFieldfrom documents.models import CustomFieldInstance-from documents.models import Documentfrom documents.models import DocumentTypefrom documents.models import Notefrom documents.models import StoragePathfrom documents.search._query import parse_user_query+from documents.tests.factories import DocumentFactoryif TYPE_CHECKING:+ from collections.abc import Callable++ from documents.models import Documentfrom documents.search._backend import TantivyBackendpytestmark = [pytest.mark.search, pytest.mark.django_db]@@ -32,49 +35,28 @@ pytestmark = [pytest.mark.search, pytest.mark.django_db]FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- """Create a Document and index it in one step, for the common case- where nothing needs to happen between the two (no related Note/- CustomFieldInstance to attach first)."""- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc--@pytest.fixture-def indexed_documents(backend: TantivyBackend) -> dict[str, int]:+def indexed_documents(index_document: Callable[..., Document]) -> dict[str, int]:"""Index a small fixture set, return {label: doc_id} for corpus queries."""docs = {- "invoice_2020": _index(- backend,+ "invoice_2020": index_document(title="Invoice 2020",content="invoice total due",- checksum="acc-invoice-2020",archive_serial_number=100,),- "invoice_2021": _index(- backend,+ "invoice_2021": index_document(title="Invoice 2021",content="invoice total due",- checksum="acc-invoice-2021",archive_serial_number=101,),- "invoice_2023": _index(- backend,+ "invoice_2023": index_document(title="Invoice 2023",content="invoice total due",- checksum="acc-invoice-2023",archive_serial_number=102,),- "receipt_2022": _index(- backend,+ "receipt_2022": index_document(title="Receipt 2022",content="receipt total due",- checksum="acc-receipt-2022",archive_serial_number=103,),}@@ -87,7 +69,7 @@ class TestIssue13568BracketWildcard:def test_bracket_class_wildcard_matches_only_in_range_years(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_documents: dict[str, int],) -> None:"""@@ -105,7 +87,7 @@ class TestIssue13568BracketWildcard:- Only the 2020 and 2021 documents match, proving the bracketcharacter class survived (issue #13568's original bug)"""- matched = _matched_ids(backend, "title:202[0-1]*")+ matched = matched_ids("title:202[0-1]*")expected = {indexed_documents["invoice_2020"],indexed_documents["invoice_2021"],@@ -121,6 +103,7 @@ class TestFieldBoosts:def test_title_boost_ranks_title_match_above_content_only_match(self,backend: TantivyBackend,+ index_document: Callable[..., Document],) -> None:"""GIVEN:@@ -132,17 +115,13 @@ class TestFieldBoosts:- The title match ranks first, proving our title field boostactually affects ranking"""- title_match = _index(- backend,+ title_match = index_document(title="urgent",content="nothing else relevant",- checksum="acc-boost-title",)- _index(- backend,+ index_document(title="nothing",content="urgent matter here",- checksum="acc-boost-content",)query = parse_user_query(backend._index, "urgent", UTC)searcher = backend._index.searcher()@@ -157,6 +136,8 @@ class TestJsonSubpaths:def test_notes_user_matches_document_with_that_note_author(self,backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -168,20 +149,20 @@ class TestJsonSubpaths:- Only the document with alice's note matches"""alice = User.objects.create_user(username="alice")- doc_with_note = Document.objects.create(+ doc_with_note = DocumentFactory(title="Has note",content="x",- checksum="acc-note-with",)Note.objects.create(document=doc_with_note, user=alice, note="reminder")backend.add_or_update(doc_with_note)- _index(backend, title="No note", content="x", checksum="acc-note-without")- matched = _matched_ids(backend, "notes.user:alice")+ index_document(title="No note", content="x")+ matched = matched_ids("notes.user:alice")assert matched == {doc_with_note.pk}def test_custom_fields_name_and_value_combine(self,backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -203,10 +184,9 @@ class TestJsonSubpaths:name="Other Field",data_type=CustomField.FieldDataType.STRING,)- matching = Document.objects.create(+ matching = DocumentFactory(title="Matching",content="x",- checksum="acc-cf-matching",)CustomFieldInstance.objects.create(document=matching,@@ -214,10 +194,9 @@ class TestJsonSubpaths:value_text="policy",)backend.add_or_update(matching)- non_matching = Document.objects.create(+ non_matching = DocumentFactory(title="Non-matching",content="x",- checksum="acc-cf-nonmatching",)CustomFieldInstance.objects.create(document=non_matching,@@ -225,8 +204,7 @@ class TestJsonSubpaths:value_text="policy",)backend.add_or_update(non_matching)- matched = _matched_ids(- backend,+ matched = matched_ids('custom_fields.name:"Contract Number" custom_fields.value:policy',)assert matched == {matching.pk}@@ -240,7 +218,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText:def test_tag_id_query_matches_nothing(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_documents: dict[str, int],) -> None:"""@@ -254,7 +232,7 @@ class TestUnregisteredIdFieldFoldsToLiteralText:- It folds to a literal text search and matches nothing,rather than erroring"""- matched = _matched_ids(backend, "tag_id:5")+ matched = matched_ids("tag_id:5")assert matched == set()@@ -268,7 +246,8 @@ class TestFuzzyBlendSurvivesWhooshGrammar:def test_typo_fuzzy_matches_alongside_date_keyword(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],settings,) -> None:"""@@ -287,26 +266,25 @@ class TestFuzzyBlendSurvivesWhooshGrammar:"""settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5with time_machine.travel(FROZEN_NOW, tick=False):- doc = _index(- backend,+ doc = index_document(title="Receipt March",content="receipt total due",- checksum="fuzzy-blend-1",archive_serial_number=900,)# Sanity: the exact spelling matches through the exact clause.- assert doc.pk in _matched_ids(backend, "added:today receipt")+ assert doc.pk in matched_ids("added:today receipt")# The regression: the misspelling (one transposition) only# matches via the fuzzy clause, and "added:today" is# whoosh-only grammar tantivy's parser rejects, so raw-string# fuzzy parsing skips the clause entirely and this returns# nothing. The typo is deliberate; keep codespell away from it.typo_query = "added:today reciept" # codespell:ignore reciept- assert doc.pk in _matched_ids(backend, typo_query)+ assert doc.pk in matched_ids(typo_query)def test_negated_words_do_not_fuzzy_match(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],settings,) -> None:"""@@ -329,14 +307,12 @@ class TestFuzzyBlendSurvivesWhooshGrammar:"""settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.5with time_machine.travel(FROZEN_NOW, tick=False):- _index(- backend,+ index_document(title="Receipt Archive",content="receipt archived stack",- checksum="fuzzy-blend-2",archive_serial_number=901,)- assert _matched_ids(backend, "added:today total NOT receipt") == set()+ assert matched_ids("added:today total NOT receipt") == set()class TestUnquotedDateKeywordPhrases:@@ -346,21 +322,20 @@ class TestUnquotedDateKeywordPhrases:spelling keeps working now that paperless no longer pre-quotes it."""@pytest.fixture- def period_documents(self, backend: TantivyBackend) -> dict[str, int]:+ def period_documents(+ self,+ index_document: Callable[..., Document],+ ) -> dict[str, int]:with time_machine.travel(FROZEN_NOW, tick=False):- in_may = _index(- backend,+ in_may = index_document(title="May Doc",content="statement",- checksum="kw-may",archive_serial_number=910,added=datetime(2026, 5, 20, 12, 0, tzinfo=UTC),)- in_june = _index(- backend,+ in_june = index_document(title="June Doc",content="statement",- checksum="kw-june",archive_serial_number=911,added=datetime(2026, 6, 10, 12, 0, tzinfo=UTC),)@@ -376,7 +351,7 @@ class TestUnquotedDateKeywordPhrases:)def test_unquoted_matches_the_same_documents_as_quoted(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],period_documents: dict[str, int],query: str,) -> None:@@ -393,7 +368,7 @@ class TestUnquotedDateKeywordPhrases:whoosh-compat's own grammar to accept it unquoted natively"""with time_machine.travel(FROZEN_NOW, tick=False):- assert _matched_ids(backend, query) == {period_documents["in_may"]}+ assert matched_ids(query) == {period_documents["in_may"]}@pytest.mark.parametrize("query",@@ -409,7 +384,7 @@ class TestUnquotedDateKeywordPhrases:)def test_every_phrase_and_date_field_parses_without_error(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],period_documents: dict[str, int],query: str,) -> None:@@ -426,11 +401,12 @@ class TestUnquotedDateKeywordPhrases:are whoosh-compat's own and are pinned in its own suite"""with time_machine.travel(FROZEN_NOW, tick=False):- _matched_ids(backend, query)+ matched_ids(query)def test_text_field_keyword_words_are_ordinary_text(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],period_documents: dict[str, int],) -> None:"""@@ -447,14 +423,12 @@ class TestUnquotedDateKeywordPhrases:documents do not match"""with time_machine.travel(FROZEN_NOW, tick=False):- wordy = _index(- backend,+ wordy = index_document(title="Notes from the previous month",content="meeting notes",- checksum="kw-text",archive_serial_number=912,)- assert _matched_ids(backend, "title:previous month") == {wordy.pk}+ assert matched_ids("title:previous month") == {wordy.pk}class TestFieldAliases:@@ -464,7 +438,8 @@ class TestFieldAliases:def test_type_alias_and_canonical_name_match_the_same_document(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -487,25 +462,22 @@ class TestFieldAliases:real index"""invoice_type = DocumentType.objects.create(name="invoice")- typed = _index(- backend,+ typed = index_document(title="First",content="quarterly statement",- checksum="alias-type-1",document_type=invoice_type,)- _index(- backend,+ index_document(title="Second",content="invoice mentioned in body",- checksum="alias-type-2",)- assert _matched_ids(backend, "type:invoice") == {typed.pk}- assert _matched_ids(backend, "document_type:invoice") == {typed.pk}+ assert matched_ids("type:invoice") == {typed.pk}+ assert matched_ids("document_type:invoice") == {typed.pk}def test_path_alias_and_canonical_name_match_the_same_document(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -523,18 +495,14 @@ class TestFieldAliases:real index"""archive = StoragePath.objects.create(name="archive", path="archive/{title}")- stored = _index(- backend,+ stored = index_document(title="Stored",content="quarterly statement",- checksum="alias-path-1",storage_path=archive,)- _index(- backend,+ index_document(title="Loose",content="archive mentioned in body",- checksum="alias-path-2",)- assert _matched_ids(backend, "path:archive") == {stored.pk}- assert _matched_ids(backend, "storage_path:archive") == {stored.pk}+ assert matched_ids("path:archive") == {stored.pk}+ assert matched_ids("storage_path:archive") == {stored.pk}diff --git a/src/documents/tests/search/test_cjk_emit_registry.py b/src/documents/tests/search/test_cjk_emit_registry.pyindex 78db2d11e..da35bf1a7 100644--- a/src/documents/tests/search/test_cjk_emit_registry.py+++ b/src/documents/tests/search/test_cjk_emit_registry.py@@ -10,19 +10,17 @@ from __future__ import annotationsimport pytestimport tantivy-import whoosh_compat as wcimport whoosh_compat.ast as wc_astfrom documents.search._query import _CJK_BIGRAM_FIELDSfrom documents.search._query import _get_emit_field_registryfrom documents.search._schema import field_descriptorsfrom documents.search._tokenizer import register_tokenizers+from documents.tests.search._ast_helpers import BIGRAM_CONTENT+from documents.tests.search._ast_helpers import CONTENTpytestmark = pytest.mark.search-_BIGRAM_CONTENT = wc.FieldRef("bigram_content")-_CONTENT = wc.FieldRef("content")-class TestEmitFieldRegistry:def test_a_multi_bigram_run_requires_every_bigram_even_under_an_or(@@ -41,8 +39,8 @@ class TestEmitFieldRegistry:"""tree = wc_ast.Or(children=(- wc_ast.Term(field=_BIGRAM_CONTENT, text="東京都"),- wc_ast.Term(field=_CONTENT, text="report"),+ wc_ast.Term(field=BIGRAM_CONTENT, text="東京都"),+ wc_ast.Term(field=CONTENT, text="report"),),)@@ -52,11 +50,11 @@ class TestEmitFieldRegistry:children=(wc_ast.And(children=(- wc_ast.Term(field=_BIGRAM_CONTENT, text="東京"),- wc_ast.Term(field=_BIGRAM_CONTENT, text="京都"),+ wc_ast.Term(field=BIGRAM_CONTENT, text="東京"),+ wc_ast.Term(field=BIGRAM_CONTENT, text="京都"),),),- wc_ast.Term(field=_CONTENT, text="report"),+ wc_ast.Term(field=CONTENT, text="report"),),)@@ -95,7 +93,7 @@ class TestEmitFieldRegistry:writer.commit()index.reload()- resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)+ resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)assert resolved is not Nonetokens = resolved.spec.analyzer(text)diff --git a/src/documents/tests/search/test_cjk_leaf_rewrite.py b/src/documents/tests/search/test_cjk_leaf_rewrite.pyindex 815ec5d22..c98a774ac 100644--- a/src/documents/tests/search/test_cjk_leaf_rewrite.py+++ b/src/documents/tests/search/test_cjk_leaf_rewrite.py@@ -19,26 +19,19 @@ from documents.search._query import _FIELD_BOOSTSfrom documents.search._query import _get_emit_field_registryfrom documents.search._query import _widen_leaffrom documents.search._registry import get_field_registry+from documents.tests.search._ast_helpers import BIGRAM_CONTENT+from documents.tests.search._ast_helpers import BIGRAM_TITLE+from documents.tests.search._ast_helpers import CONTENT+from documents.tests.search._ast_helpers import NOTES+from documents.tests.search._ast_helpers import TITLE+from documents.tests.search._ast_helpers import bigram+from documents.tests.search._ast_helpers import contentif TYPE_CHECKING:from collections.abc import Callablepytestmark = pytest.mark.search-_CONTENT = wc.FieldRef("content")-_TITLE = wc.FieldRef("title")-_NOTES = wc.FieldRef("notes", "note")-_BIGRAM_CONTENT = wc.FieldRef("bigram_content")-_BIGRAM_TITLE = wc.FieldRef("bigram_title")---def _content(text: str) -> wc_ast.Term:- return wc_ast.Term(field=_CONTENT, text=text)---def _bigram(text: str) -> wc_ast.Term:- return wc_ast.Term(field=_BIGRAM_CONTENT, text=text)-def _widened(original: wc_ast.Node, bigram_side: wc_ast.Node) -> wc_ast.Or:return wc_ast.Or(children=(original, bigram_side))@@ -61,9 +54,9 @@ class TestTheHook:@pytest.mark.parametrize("leaf",[- pytest.param(_content("invoice"), id="latin_term"),+ pytest.param(content("invoice"), id="latin_term"),pytest.param(- wc_ast.Term(field=_NOTES, text="東京"),+ wc_ast.Term(field=NOTES, text="東京"),id="non_default_field",),pytest.param(wc_ast.Term(field=None, text="東京"), id="unfielded"),@@ -96,7 +89,7 @@ class TestTheHook:- Neither cjk_terms nor latin_terms gains a piece, and theleaf is returned unchanged rather than becoming an empty Or"""- leaf = _content("⺀")+ leaf = content("⺀")assert _widen_cjk_leaf(leaf) is leaf@@ -111,7 +104,7 @@ class TestTheHook:copy: analyze() only keeps the leaf's own enclosing-groupanalysis for that exact object"""- leaf = _content("東京")+ leaf = content("東京")widened = _widen_cjk_leaf(leaf)@@ -128,11 +121,11 @@ class TestTheHook:- The bigram side targets bigram_title, the leaf's own field'scompanion, so a fielded term stays fielded"""- leaf = wc_ast.Term(field=_TITLE, text="東京")+ leaf = wc_ast.Term(field=TITLE, text="東京")assert _widen_cjk_leaf(leaf) == _widened(leaf,- wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),+ wc_ast.Term(field=BIGRAM_TITLE, text="東京"),)def test_each_cjk_run_gets_its_own_bigram_leaf(self) -> None:@@ -151,11 +144,11 @@ class TestTheHook:any one of its runs, which is what the separate bigramclause did before this change"""- leaf = _content("東京・大阪")+ leaf = content("東京・大阪")assert _widen_cjk_leaf(leaf) == _widened(leaf,- wc_ast.Or(children=(_bigram("東京"), _bigram("大阪"))),+ wc_ast.Or(children=(bigram("東京"), bigram("大阪"))),)def test_each_run_keeps_its_own_bigrams_required(self) -> None:@@ -172,11 +165,11 @@ class TestTheHook:it does not inherit from the enclosing group. Without this,東京都 would match a document holding only 京都"""- assert _analyze(_content("東京都・大阪府")) == wc_ast.Or(+ assert _analyze(content("東京都・大阪府")) == wc_ast.Or(children=(- wc_ast.And(children=(_content("東京都"), _content("大阪府"))),- wc_ast.And(children=(_bigram("東京"), _bigram("京都"))),- wc_ast.And(children=(_bigram("大阪"), _bigram("阪府"))),+ wc_ast.And(children=(content("東京都"), content("大阪府"))),+ wc_ast.And(children=(bigram("東京"), bigram("京都"))),+ wc_ast.And(children=(bigram("大阪"), bigram("阪府"))),),)@@ -191,9 +184,9 @@ class TestTheHook:every token at position 0, so a positional Phrase against itcould never match"""- leaf = wc_ast.Phrase(field=_CONTENT, text="東京都")+ leaf = wc_ast.Phrase(field=CONTENT, text="東京都")- assert _widen_cjk_leaf(leaf) == _widened(leaf, _bigram("東京都"))+ assert _widen_cjk_leaf(leaf) == _widened(leaf, bigram("東京都"))def test_a_multi_word_phrase_requires_every_run(self) -> None:"""@@ -208,27 +201,27 @@ class TestTheHook:the quoted phrase match strictly more than the same twowords unquoted, and quoting must not widen a search"""- leaf = wc_ast.Phrase(field=_CONTENT, text="東京都 大阪府")+ leaf = wc_ast.Phrase(field=CONTENT, text="東京都 大阪府")assert _widen_cjk_leaf(leaf) == _widened(leaf,- wc_ast.And(children=(_bigram("東京都"), _bigram("大阪府"))),+ wc_ast.And(children=(bigram("東京都"), bigram("大阪府"))),)@pytest.mark.parametrize(("leaf", "alternative"),[pytest.param(- _content("東京-report"),- wc_ast.And(children=(_bigram("東京"), _content("report"))),+ content("東京-report"),+ wc_ast.And(children=(bigram("東京"), content("report"))),id="separated_term",),pytest.param(- wc_ast.Phrase(field=_CONTENT, text="東京 report"),- wc_ast.And(children=(_bigram("東京"), _content("report"))),+ wc_ast.Phrase(field=CONTENT, text="東京 report"),+ wc_ast.And(children=(bigram("東京"), content("report"))),id="phrase",),- pytest.param(_content("東京report"), _bigram("東京"), id="glued"),+ pytest.param(content("東京report"), bigram("東京"), id="glued"),],)def test_latin_is_required_only_where_the_analyzer_splits_it_off(@@ -265,9 +258,9 @@ class TestTheHook:in And (a single piece collapses to itself), even thoughthe CJK side contributed nothing"""- leaf = _content("⺀report")+ leaf = content("⺀report")- assert _widen_cjk_leaf(leaf) == _widened(leaf, _content("report"))+ assert _widen_cjk_leaf(leaf) == _widened(leaf, content("report"))def test_new_nodes_carry_the_leaf_span(self) -> None:"""@@ -280,7 +273,7 @@ class TestTheHook:span, so an emit-time diagnostic still points into thequery text"""- leaf = wc_ast.Term(field=_CONTENT, text="東京・大阪", startchar=3, endchar=8)+ leaf = wc_ast.Term(field=CONTENT, text="東京・大阪", startchar=3, endchar=8)widened = _widen_cjk_leaf(leaf)@@ -298,9 +291,9 @@ class TestAnalyzedTree:@pytest.mark.parametrize("node",[- pytest.param(wc_ast.Prefix(field=_CONTENT, text="東京"), id="cjk_prefix"),+ pytest.param(wc_ast.Prefix(field=CONTENT, text="東京"), id="cjk_prefix"),pytest.param(- wc_ast.Wildcard(field=_CONTENT, pattern="東*"),+ wc_ast.Wildcard(field=CONTENT, pattern="東*"),id="cjk_wildcard",),],@@ -340,13 +333,13 @@ class TestAnalyzedTree:still negated under NOT. A NOT left holding nothing wouldturn into "match everything" instead"""- resolved = _get_emit_field_registry(None).resolve(_BIGRAM_CONTENT)+ resolved = _get_emit_field_registry(None).resolve(BIGRAM_CONTENT)assert resolved is not Nonebigrams = resolved.spec.analyzer(long_cjk_run)- assert _analyze(build(_content(long_cjk_run))) == build(+ assert _analyze(build(content(long_cjk_run))) == build(wc_ast.And(- children=tuple(_bigram(token) for token in dict.fromkeys(bigrams)),+ children=tuple(bigram(token) for token in dict.fromkeys(bigrams)),),)@@ -361,11 +354,11 @@ class TestAnalyzedTree:has no bigram. An accepted gap, the same on the positive andthe negated side"""- analyzed = _analyze(_content("東・大阪"))+ analyzed = _analyze(content("東・大阪"))assert analyzed == _widened(- wc_ast.And(children=(_content("東"), _content("大阪"))),- _bigram("大阪"),+ wc_ast.And(children=(content("東"), content("大阪"))),+ bigram("大阪"),)def test_the_title_boost_wraps_the_widened_title_leaf(self) -> None:@@ -393,14 +386,14 @@ class TestAnalyzedTree:assert (wc_ast.Boosted(child=_widened(- wc_ast.Term(field=_TITLE, text="東京"),- wc_ast.Term(field=_BIGRAM_TITLE, text="東京"),+ wc_ast.Term(field=TITLE, text="東京"),+ wc_ast.Term(field=BIGRAM_TITLE, text="東京"),),boost=_FIELD_BOOSTS["title"],)in analyzed.children)- assert {_BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {+ assert {BIGRAM_CONTENT, wc.FieldRef("bigram_tag")} <= {child.field for child in analyzed.children if isinstance(child, wc_ast.Term)}@@ -418,18 +411,18 @@ class TestAnalyzedTree:alternative is not a single child here and _widened() doesnot fit; the tree is written out literally"""- tree = wc_ast.And(children=(_content("東京・大阪"), _content("report")))+ tree = wc_ast.And(children=(content("東京・大阪"), content("report")))assert _analyze(tree) == wc_ast.And(children=(wc_ast.Or(children=(- wc_ast.And(children=(_content("東京"), _content("大阪"))),- _bigram("東京"),- _bigram("大阪"),+ wc_ast.And(children=(content("東京"), content("大阪"))),+ bigram("東京"),+ bigram("大阪"),),),- _content("report"),+ content("report"),),)@@ -443,15 +436,15 @@ class TestAnalyzedTree:- The original side ORs its tokens, as the group says, and thehook's Or flattens into the group"""- tree = wc_ast.Or(children=(_content("東京・大阪"), _content("report")))+ tree = wc_ast.Or(children=(content("東京・大阪"), content("report")))assert _analyze(tree) == wc_ast.Or(children=(- _content("東京"),- _content("大阪"),- _bigram("東京"),- _bigram("大阪"),- _content("report"),+ content("東京"),+ content("大阪"),+ bigram("東京"),+ bigram("大阪"),+ content("report"),),)@@ -460,12 +453,12 @@ class TestAnalyzedTree:[pytest.param(lambda leaf: wc_ast.Not(child=leaf), id="not"),pytest.param(- lambda leaf: wc_ast.AndNot(positive=_content("invoice"), negative=leaf),+ lambda leaf: wc_ast.AndNot(positive=content("invoice"), negative=leaf),id="andnot",),pytest.param(lambda leaf: wc_ast.AndMaybe(- required=_content("invoice"),+ required=content("invoice"),optional=leaf,),id="andmaybe",@@ -473,7 +466,7 @@ class TestAnalyzedTree:pytest.param(lambda leaf: wc_ast.Require(scored=leaf,- filter_only=_content("invoice"),+ filter_only=content("invoice"),),id="require",),@@ -500,6 +493,6 @@ class TestAnalyzedTree:position. A negated CJK leaf is widened too, so a negationexcludes exactly what the positive search would match"""- assert _analyze(build(_content("東京"))) == build(- _widened(_content("東京"), _bigram("東京")),+ assert _analyze(build(content("東京"))) == build(+ _widened(content("東京"), bigram("東京")),)diff --git a/src/documents/tests/search/test_compact_date_forms.py b/src/documents/tests/search/test_compact_date_forms.pyindex 48e0f2fd2..8ad4a54f5 100644--- a/src/documents/tests/search/test_compact_date_forms.py+++ b/src/documents/tests/search/test_compact_date_forms.py@@ -20,67 +20,53 @@ from typing import TYPE_CHECKINGimport pytest-from documents.models import Document-if TYPE_CHECKING:- from documents.search._backend import TantivyBackend--pytestmark = [pytest.mark.search, pytest.mark.django_db]+ from collections.abc import Callable+ from documents.models import Document-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc+pytestmark = [pytest.mark.search, pytest.mark.django_db]@pytest.fixture-def docs(backend: TantivyBackend) -> dict[str, int]:+def docs(index_document: Callable[..., Document]) -> dict[str, int]:return {- "instant": _index(- backend,+ "instant": index_document(title="On the instant",content="x",- checksum="compact-date-instant",added=datetime(2005, 3, 4, 15, 30, tzinfo=UTC),).pk,- "same_day": _index(- backend,+ "same_day": index_document(title="Same day, other hour",content="x",- checksum="compact-date-same-day",added=datetime(2005, 3, 4, 9, 0, tzinfo=UTC),).pk,- "next_day": _index(- backend,+ "next_day": index_document(title="Next day, same hour",content="x",- checksum="compact-date-next-day",added=datetime(2005, 3, 5, 15, 30, tzinfo=UTC),).pk,}-def test_fourteen_digits_is_a_single_instant(- backend: TantivyBackend,- docs: dict[str, int],-) -> None:- """- GIVEN:- - Three documents indexed on the ``added`` DATETIME fast field:- one at 2005-03-04T15:30:00, one on the same calendar day at a- different hour, and one on the next day at the same hour- WHEN:- - Searching with the 14-digit compact date form- ``added:20050304153000``- THEN:- - Only the document at that exact instant matches; the same-day- document is what tells this apart from the 8-digit day-window- form, and the next-day document from a form that ignored the- time of day altogether- """- assert _matched_ids(backend, "added:20050304153000") == {docs["instant"]}+class TestCompactDateForms:+ def test_fourteen_digits_is_a_single_instant(+ self,+ matched_ids: Callable[[str], set[int]],+ docs: dict[str, int],+ ) -> None:+ """+ GIVEN:+ - Three documents indexed on the ``added`` DATETIME fast field:+ one at 2005-03-04T15:30:00, one on the same calendar day at a+ different hour, and one on the next day at the same hour+ WHEN:+ - Searching with the 14-digit compact date form+ ``added:20050304153000``+ THEN:+ - Only the document at that exact instant matches; the same-day+ document is what tells this apart from the 8-digit day-window+ form, and the next-day document from a form that ignored the+ time of day altogether+ """+ assert matched_ids("added:20050304153000") == {docs["instant"]}diff --git a/src/documents/tests/search/test_date_keyword_phrase_removal.py b/src/documents/tests/search/test_date_keyword_phrase_removal.pyindex 97b814351..7dbd7998b 100644--- a/src/documents/tests/search/test_date_keyword_phrase_removal.py+++ b/src/documents/tests/search/test_date_keyword_phrase_removal.py@@ -19,22 +19,12 @@ from typing import TYPE_CHECKINGimport pytest-from documents.models import Document-if TYPE_CHECKING:- from documents.search._backend import TantivyBackend--pytestmark = [pytest.mark.search, pytest.mark.django_db]+ from collections.abc import Callable+ from documents.models import Document-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc+pytestmark = [pytest.mark.search, pytest.mark.django_db]class TestQuotedStringContainingDateKeywordText:@@ -46,7 +36,8 @@ class TestQuotedStringContainingDateKeywordText:def test_matches_only_the_literal_phrase(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -67,11 +58,9 @@ class TestQuotedStringContainingDateKeywordText:never spill into an unfielded search for "previous" and"month" across the default search fields"""- literal = _index(- backend,+ literal = index_document(title="see added:previous month notes",content="quarterly filing",- checksum="dkp-literal",archive_serial_number=920,)# Under the deleted rewrite, this decoy would incorrectly match:@@ -79,13 +68,11 @@ class TestQuotedStringContainingDateKeywordText:# corrupted parse required as title phrases, and its content# supplies "previous" and "month" as the decomposed word-match# clauses the rewrite turned the middle of the phrase into.- decoy = _index(- backend,+ decoy = index_document(title="see added: quarterly report notes",content="we reviewed the previous statement about month end",- checksum="dkp-decoy",archive_serial_number=921,)query = 'title:"see added:previous month notes"'- assert _matched_ids(backend, query) == {literal.pk}- assert decoy.pk not in _matched_ids(backend, query)+ assert matched_ids(query) == {literal.pk}+ assert decoy.pk not in matched_ids(query)diff --git a/src/documents/tests/search/test_date_keyword_timezone.py b/src/documents/tests/search/test_date_keyword_timezone.pyindex 73eca2bbe..1071ccaec 100644--- a/src/documents/tests/search/test_date_keyword_timezone.py+++ b/src/documents/tests/search/test_date_keyword_timezone.py@@ -32,32 +32,23 @@ from typing import TYPE_CHECKINGimport pytestimport time_machine-from documents.models import Document-if TYPE_CHECKING:+ from collections.abc import Callable+from pytest_django.fixtures import SettingsWrapper- from documents.search._backend import TantivyBackend+ from documents.models import Documentpytestmark = [pytest.mark.search, pytest.mark.django_db]FROZEN_NOW = datetime(2026, 6, 15, 2, 0, tzinfo=UTC)-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc--class TestDateKeywordUsesTheActiveTimezone:def test_today_matches_the_new_york_calendar_day_not_the_utc_one(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],settings: SettingsWrapper,) -> None:"""@@ -79,22 +70,18 @@ class TestDateKeywordUsesTheActiveTimezone:"""settings.TIME_ZONE = "America/New_York"with time_machine.travel(FROZEN_NOW, tick=False):- in_ny_today = _index(- backend,+ in_ny_today = index_document(title="NY today",content="x",- checksum="tz-keyword-ny-today",added=datetime(2026, 6, 14, 20, 0, tzinfo=UTC),)# Not captured: the exact-set assertion below already proves# this document (inside a naive UTC-calendar-day window, but# outside New York's actual "today") does not match.- _index(- backend,+ index_document(title="UTC calendar day only",content="x",- checksum="tz-keyword-utc-calendar-day-only",added=datetime(2026, 6, 15, 10, 0, tzinfo=UTC),)- assert _matched_ids(backend, "added:today") == {in_ny_today.pk}+ assert matched_ids("added:today") == {in_ny_today.pk}diff --git a/src/documents/tests/search/test_documented_syntax.py b/src/documents/tests/search/test_documented_syntax.pyindex fa69f3f68..d00551c25 100644--- a/src/documents/tests/search/test_documented_syntax.py+++ b/src/documents/tests/search/test_documented_syntax.py@@ -24,16 +24,18 @@ from typing import TYPE_CHECKINGimport pytestimport time_machine-from documents.models import Documentfrom documents.models import Notefrom documents.models import Tagfrom documents.search._errors import InvalidDateQuery+from documents.tests.factories import DocumentFactoryif TYPE_CHECKING:+ from collections.abc import Callablefrom collections.abc import Generatorfrom django.contrib.auth.models import User+ from documents.models import Documentfrom documents.search._backend import TantivyBackendpytestmark = [pytest.mark.search, pytest.mark.django_db]@@ -45,37 +47,23 @@ FROZEN_NOW = datetime(2026, 6, 15, 12, 0, tzinfo=UTC)DOC_CHECKSUM = "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc--class TestLogicalExpressions:@pytest.fixture- def docs(self, backend: TantivyBackend) -> dict[str, int]:+ def docs(self, index_document: Callable[..., Document]) -> dict[str, int]:return {- "secret": _index(- backend,+ "secret": index_document(title="Invoice one",content="invoice secret contents",- checksum="doc-syntax-secret",).pk,- "plain": _index(- backend,+ "plain": index_document(title="Invoice two",content="invoice ordinary contents",- checksum="doc-syntax-plain",).pk,}def test_not_excludes_a_term(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],docs: dict[str, int],) -> None:"""@@ -86,11 +74,11 @@ class TestLogicalExpressions:THEN:- Only the document without "secret" matches"""- assert _matched_ids(backend, "invoice NOT secret") == {docs["plain"]}+ assert matched_ids("invoice NOT secret") == {docs["plain"]}def test_leading_hyphen_requires_the_term_instead_of_excluding_it(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],docs: dict[str, int],) -> None:"""@@ -104,11 +92,11 @@ class TestLogicalExpressions:indexed as the plain term "secret" and the query becomes anAND rather than an exclusion, exactly as the docs warn"""- assert _matched_ids(backend, "invoice -secret") == {docs["secret"]}+ assert matched_ids("invoice -secret") == {docs["secret"]}def test_or_inside_parentheses_matches_either_branch(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],docs: dict[str, int],) -> None:"""@@ -120,14 +108,15 @@ class TestLogicalExpressions:THEN:- Both documents match"""- matched = _matched_ids(backend, "invoice AND (secret OR ordinary)")+ matched = matched_ids("invoice AND (secret OR ordinary)")assert matched == {docs["secret"], docs["plain"]}class TestPhraseSearch:def test_quoted_phrase_requires_the_words_in_order(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -138,14 +127,12 @@ class TestPhraseSearch:- The in-order phrase matches, and the same words reordered donot"""- doc = _index(- backend,+ doc = index_document(title="Phrase",content="the quick brown fox jumps",- checksum="doc-syntax-phrase",)- assert _matched_ids(backend, '"quick brown fox"') == {doc.pk}- assert _matched_ids(backend, '"brown quick fox"') == set()+ assert matched_ids('"quick brown fox"') == {doc.pk}+ assert matched_ids('"brown quick fox"') == set()class TestTagCommaList:@@ -166,6 +153,7 @@ class TestTagCommaList:def test_comma_list_requires_every_listed_tag(self,backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -181,30 +169,22 @@ class TestTagCommaList:unpaid = Tag.objects.create(name="unpaid")archived = Tag.objects.create(name="archived")- both = Document.objects.create(- title="Both tags",- content="body",- checksum="doc-syntax-tag-both",- )+ both = DocumentFactory(title="Both tags", content="body")both.tags.add(bills, unpaid)backend.add_or_update(both)- one = Document.objects.create(- title="One tag",- content="body",- checksum="doc-syntax-tag-one",- )+ one = DocumentFactory(title="One tag", content="body")one.tags.add(bills, archived)backend.add_or_update(one)- assert _matched_ids(backend, "tag:bills,unpaid") == {both.pk}- assert _matched_ids(backend, "tag:bills") == {both.pk, one.pk}+ assert matched_ids("tag:bills,unpaid") == {both.pk}+ assert matched_ids("tag:bills") == {both.pk, one.pk}class TestArchiveMetadataFields:@pytest.fixturedef doc(self, backend: TantivyBackend, admin_user: User) -> Document:- doc = Document.objects.create(+ doc = DocumentFactory(title="Metadata",content="body",checksum=DOC_CHECKSUM,@@ -237,7 +217,7 @@ class TestArchiveMetadataFields:)def test_documented_metadata_query_matches(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],doc: Document,query: str,) -> None:@@ -253,7 +233,7 @@ class TestArchiveMetadataFields:THEN:- Each one matches the document"""- assert _matched_ids(backend, query) == {doc.pk}+ assert matched_ids(query) == {doc.pk}@pytest.mark.parametrize("query",@@ -265,7 +245,7 @@ class TestArchiveMetadataFields:)def test_partial_or_uppercase_checksum_matches_nothing(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],doc: Document,query: str,) -> None:@@ -279,7 +259,7 @@ class TestArchiveMetadataFields:- Nothing matches, as the docs say only a complete, lowercasechecksum matches as an exact value"""- assert _matched_ids(backend, query) == set()+ assert matched_ids(query) == set()class TestDocumentedDateForms:@@ -289,7 +269,7 @@ class TestDocumentedDateForms:yield@pytest.fixture- def dated(self, backend: TantivyBackend) -> dict[str, int]:+ def dated(self, index_document: Callable[..., Document]) -> dict[str, int]:stamps = {"today": datetime(2026, 6, 15, 9, 0, tzinfo=UTC),"yesterday": datetime(2026, 6, 14, 9, 0, tzinfo=UTC),@@ -300,11 +280,9 @@ class TestDocumentedDateForms:"old": datetime(2005, 3, 4, 15, 30, tzinfo=UTC),}return {- label: _index(- backend,+ label: index_document(title=label,content="dated body",- checksum=f"doc-syntax-date-{label}",added=stamp,).pkfor label, stamp in stamps.items()@@ -335,7 +313,7 @@ class TestDocumentedDateForms:)def test_documented_date_form_matches_its_day_or_month(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],dated: dict[str, int],query: str,label: str,@@ -355,7 +333,7 @@ class TestDocumentedDateForms:- Each form matches exactly the document dated on its day orwithin its month"""- assert _matched_ids(backend, query) == {dated[label]}+ assert matched_ids(query) == {dated[label]}@pytest.mark.parametrize("query",@@ -381,7 +359,7 @@ class TestDocumentedDateForms:)def test_forms_the_docs_warn_about_match_nothing(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],dated: dict[str, int],query: str,) -> None:@@ -400,11 +378,11 @@ class TestDocumentedDateForms:- Nothing matches, exactly as the docs warn, rather thanpresenting these as usable spellings"""- assert _matched_ids(backend, query) == set()+ assert matched_ids(query) == set()def test_bare_timestamp_is_rejected_rather_than_matching_nothing(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],dated: dict[str, int],) -> None:"""@@ -423,13 +401,13 @@ class TestDocumentedDateForms:prefix the date grammar's tokenizer first split on"""with pytest.raises(InvalidDateQuery) as exc_info:- _matched_ids(backend, "added:2005-03-04T15:30:00Z")+ matched_ids("added:2005-03-04T15:30:00Z")assert exc_info.value.field == "added"assert exc_info.value.value == "2005-03-04T15:30:00Z"def test_relative_offset_as_a_range_bound_is_a_real_window(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],dated: dict[str, int],) -> None:"""@@ -447,14 +425,14 @@ class TestDocumentedDateForms:the offset itself and not a whole-day rounding of it, asthe docs say next to the warning about the standalone form"""- assert _matched_ids(backend, "added:['-1 week' to now]") == {+ assert matched_ids("added:['-1 week' to now]") == {dated["today"],dated["yesterday"],}def test_double_quoted_range_bound_is_rejected(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],dated: dict[str, int],) -> None:"""@@ -471,5 +449,5 @@ class TestDocumentedDateForms:attached and is not a recognizable date"""with pytest.raises(InvalidDateQuery) as exc_info:- _matched_ids(backend, 'added:["2005-03-04" to 2005-03-05]')+ matched_ids('added:["2005-03-04" to 2005-03-05]')assert exc_info.value.value == '"2005-03-04"'diff --git a/src/documents/tests/search/test_error_routing.py b/src/documents/tests/search/test_error_routing.pyindex cd8822560..3382a395e 100644--- a/src/documents/tests/search/test_error_routing.py+++ b/src/documents/tests/search/test_error_routing.py@@ -8,9 +8,9 @@ from __future__ import annotationsimport loggingfrom datetime import UTC+from typing import TYPE_CHECKINGimport pytest-import tantivyfrom whoosh_compat.errors import Diagnosticfrom whoosh_compat.errors import DiagnosticKindfrom whoosh_compat.errors import QueryError@@ -22,20 +22,14 @@ from documents.search._errors import SearchQueryErrorfrom documents.search._query import _map_emit_errorfrom documents.search._query import _single_diagnostic_to_errorfrom documents.search._query import parse_user_query-from documents.search._schema import build_schema-from documents.search._tokenizer import register_tokenizers-pytestmark = pytest.mark.search+if TYPE_CHECKING:+ import tantivy-_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"+pytestmark = pytest.mark.search[email protected](scope="module")-def query_index() -> tantivy.Index:- """An in-memory, unstemmed index; these tests only parse, never index."""- idx = tantivy.Index(build_schema(), path=None)- register_tokenizers(idx, "")- return idx+_LIBRARY_PROSE = "INTERNAL LIBRARY WORDING WITH raw tantivy detail"def _diagnostic(diff --git a/src/documents/tests/search/test_exists_on_json_fields.py b/src/documents/tests/search/test_exists_on_json_fields.pyindex 875087439..945668f86 100644--- a/src/documents/tests/search/test_exists_on_json_fields.py+++ b/src/documents/tests/search/test_exists_on_json_fields.py@@ -19,9 +19,9 @@ from __future__ import annotationsimport loggingfrom datetime import UTC+from typing import TYPE_CHECKINGimport pytest-import tantivyfrom whoosh_compat.errors import Diagnosticfrom whoosh_compat.errors import DiagnosticKindfrom whoosh_compat.errors import QueryError@@ -32,8 +32,10 @@ from whoosh_compat.fields import FieldReffrom documents.search._errors import SearchQueryErrorfrom documents.search._query import _map_emit_errorfrom documents.search._query import parse_user_query-from documents.search._schema import build_schema-from documents.search._tokenizer import register_tokenizers++if TYPE_CHECKING:+ import tantivy+pytestmark = pytest.mark.search@@ -48,13 +50,6 @@ EXISTS_QUERIES = [][email protected](scope="module")-def query_index() -> tantivy.Index:- idx = tantivy.Index(build_schema(), path=None)- register_tokenizers(idx, "")- return idx--class TestJsonExistsIsUserError:@pytest.mark.parametrize("query", EXISTS_QUERIES)def test_query_is_a_400_that_emits_no_error_log(diff --git a/src/documents/tests/search/test_fuzzy_alternative.py b/src/documents/tests/search/test_fuzzy_alternative.pyindex b788bc7c4..0e81893e9 100644--- a/src/documents/tests/search/test_fuzzy_alternative.py+++ b/src/documents/tests/search/test_fuzzy_alternative.py@@ -14,11 +14,12 @@ import whoosh_compat as wcimport whoosh_compat.ast as wc_astfrom documents.search._query import _fuzzy_alternative+from documents.tests.search._ast_helpers import CONTENT+from documents.tests.search._ast_helpers import TITLE+from documents.tests.search._ast_helpers import contentpytestmark = pytest.mark.search-_CONTENT = wc.FieldRef("content")-_TITLE = wc.FieldRef("title")# 130 characters, past the analyzer's 129-byte remove_long limit (128 is# kept, 129 is dropped), so the index never holds it and neither side@@ -26,10 +27,6 @@ _TITLE = wc.FieldRef("title")_TOO_LONG = "x" * 130-def _content(text: str) -> wc_ast.Term:- return wc_ast.Term(field=_CONTENT, text=text)--def _fuzzy(field: wc.FieldRef, text: str) -> wc_ast.Fuzzy:return wc_ast.Fuzzy(field=field, text=text, distance=1, prefix=True)@@ -45,7 +42,7 @@ class TestTheAlternative:- It is a single Fuzzy leaf on the same field, distance 1 andprefix matching, which is what the old clause used"""- assert _fuzzy_alternative(_content("invoice")) == _fuzzy(_CONTENT, "invoice")+ assert _fuzzy_alternative(content("invoice")) == _fuzzy(CONTENT, "invoice")def test_the_leafs_own_field_is_used(self) -> None:"""@@ -58,8 +55,8 @@ class TestTheAlternative:fields. This is the fielding fix: today's clause searches afielded word everywhere"""- assert _fuzzy_alternative(wc_ast.Term(field=_TITLE, text="invoice")) == _fuzzy(- _TITLE,+ assert _fuzzy_alternative(wc_ast.Term(field=TITLE, text="invoice")) == _fuzzy(+ TITLE,"invoice",)@@ -74,8 +71,8 @@ class TestTheAlternative:side, which is OR'd for an unfielded term, and would losetoday's per-word recall inside a term"""- assert _fuzzy_alternative(_content("COVID-19")) == wc_ast.Or(- children=(_fuzzy(_CONTENT, "covid"), _fuzzy(_CONTENT, "19")),+ assert _fuzzy_alternative(content("COVID-19")) == wc_ast.Or(+ children=(_fuzzy(CONTENT, "covid"), _fuzzy(CONTENT, "19")),)def test_words_are_split_where_the_index_splits_them(self) -> None:@@ -88,7 +85,7 @@ class TestTheAlternative:- The Fuzzy text is the analyzer's output, not the raw text,so it is in the same shape as the index terms"""- assert _fuzzy_alternative(_content("Éclair")) == _fuzzy(_CONTENT, "eclair")+ assert _fuzzy_alternative(content("Éclair")) == _fuzzy(CONTENT, "eclair")def test_the_splitter_does_not_stem(self) -> None:"""@@ -101,8 +98,8 @@ class TestTheAlternative:pattern_normalizer, which stems it once; stemming here toowould search for a term the index does not hold"""- assert _fuzzy_alternative(_content("universities")) == _fuzzy(- _CONTENT,+ assert _fuzzy_alternative(content("universities")) == _fuzzy(+ CONTENT,"universities",)@@ -130,7 +127,7 @@ class TestTheAlternative:that can never match, making a query return LESS with fuzzyon than off"""- assert _fuzzy_alternative(_content(text)) is None+ assert _fuzzy_alternative(content(text)) is Nonedef test_a_one_character_word_is_dropped_from_a_longer_term(self) -> None:"""@@ -142,7 +139,7 @@ class TestTheAlternative:- Only the real word survives. A one-character prefix fuzzyterm matches every term in the field"""- assert _fuzzy_alternative(_content("h52.1")) == _fuzzy(_CONTENT, "h52")+ assert _fuzzy_alternative(content("h52.1")) == _fuzzy(CONTENT, "h52")def test_the_leaf_span_is_copied(self) -> None:"""@@ -155,7 +152,7 @@ class TestTheAlternative:- Every node it builds carries that span, so an emit-timediagnostic still points into the query text"""- leaf = wc_ast.Term(field=_CONTENT, text="ab-cd", startchar=4, endchar=9)+ leaf = wc_ast.Term(field=CONTENT, text="ab-cd", startchar=4, endchar=9)alternative = _fuzzy_alternative(leaf)@@ -181,10 +178,10 @@ class TestPhrasesAreNotWidened:quoted phrase match strictly more than the same two wordsunquoted"""- leaf = wc_ast.Phrase(field=_CONTENT, text="tax report")+ leaf = wc_ast.Phrase(field=CONTENT, text="tax report")assert _fuzzy_alternative(leaf) == wc_ast.And(- children=(_fuzzy(_CONTENT, "tax"), _fuzzy(_CONTENT, "report")),+ children=(_fuzzy(CONTENT, "tax"), _fuzzy(CONTENT, "report")),)@@ -209,7 +206,7 @@ class TestCjkGetsNoFuzzySide:one edit of its start, which is the "東京都 matches 京都"failure the bigram fields exist to avoid"""- assert _fuzzy_alternative(_content(text)) is None+ assert _fuzzy_alternative(content(text)) is Nonedef test_latin_beside_cjk_still_gets_its_fuzzy_side(self) -> None:"""@@ -221,6 +218,6 @@ class TestCjkGetsNoFuzzySide:- Only the latin word is fuzzed. Skipping CJK words must notcost the latin half its near-match"""- leaf = _content("東京 report")+ leaf = content("東京 report")- assert _fuzzy_alternative(leaf) == _fuzzy(_CONTENT, "report")+ assert _fuzzy_alternative(leaf) == _fuzzy(CONTENT, "report")diff --git a/src/documents/tests/search/test_fuzzy_tokenization.py b/src/documents/tests/search/test_fuzzy_tokenization.pyindex 8eaff30bb..e95b12df0 100644--- a/src/documents/tests/search/test_fuzzy_tokenization.py+++ b/src/documents/tests/search/test_fuzzy_tokenization.py@@ -13,38 +13,23 @@ from typing import TYPE_CHECKINGimport pytest-from documents.models import Document-if TYPE_CHECKING:- from pytest_django.fixtures import SettingsWrapper-- from documents.search._backend import TantivyBackend--pytestmark = [pytest.mark.search, pytest.mark.django_db]---def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))+ from collections.abc import Callable+ from documents.models import Document-def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc--[email protected](autouse=True)-def fuzzy_enabled(settings: SettingsWrapper) -> None:- """Enable the fuzzy blend clause. The threshold doubles as a minimum- score filter, so it is set to 0.0: every hit passes and the test sees- the clause's matching behaviour, not the filter's."""- settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0+pytestmark = [+ pytest.mark.search,+ pytest.mark.django_db,+ pytest.mark.usefixtures("fuzzy_enabled"),+]class TestFuzzyClauseWords:def test_a_stemmed_word_is_not_stemmed_a_second_time(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -62,36 +47,29 @@ class TestFuzzyClauseWords:reaches unrelated words - the clause must stay wide enoughfor a typo and no wider"""- wanted = _index(- backend,+ wanted = index_document(title="A",content="universities of europe",- checksum="fuzz-stem-1",)- typo = _index(- backend,+ typo = index_document(title="B",content="universties of europe",- checksum="fuzz-stem-2",)- _index(- backend,+ index_document(title="C",content="univalent chemical bonds",- checksum="fuzz-stem-3",)- _index(- backend,+ index_document(title="D",content="unicycle repair manual",- checksum="fuzz-stem-4",)- assert _matched_ids(backend, "universities") == {wanted.pk, typo.pk}+ assert matched_ids("universities") == {wanted.pk, typo.pk}def test_a_hyphenated_term_still_reaches_the_clause(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -105,18 +83,17 @@ class TestFuzzyClauseWords:would read as grammar, is dropped, and the whole query losesits fuzzy clause"""- misspelled = _index(- backend,+ misspelled = index_document(title="A",content="covidx testing results",- checksum="fuzz-hyphen-1",)- assert _matched_ids(backend, "COVID-19") == {misspelled.pk}+ assert matched_ids("COVID-19") == {misspelled.pk}def test_a_phrase_still_reaches_the_clause(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -129,14 +106,12 @@ class TestFuzzyClauseWords:space, and is the whole query's only free text here, so itmust still reach the clause"""- near_miss = _index(- backend,+ near_miss = index_document(title="A",content="taxation reportage weekly",- checksum="fuzz-phrase-1",)- assert _matched_ids(backend, '"tax reports"') == {near_miss.pk}+ assert matched_ids('"tax reports"') == {near_miss.pk}class TestBooleanKeywordsInRawText:@@ -146,24 +121,18 @@ class TestBooleanKeywordsInRawText:now, which closes that off structurally; these pin it shut."""@pytest.fixture- def corpus(self, backend: TantivyBackend) -> dict[str, int]:- both = _index(- backend,+ def corpus(self, index_document: Callable[..., Document]) -> dict[str, int]:+ both = index_document(title="A",content="taxation reportage weekly",- checksum="fuzz-kw-1",)- tax_only = _index(- backend,+ tax_only = index_document(title="B",content="taxation only here",- checksum="fuzz-kw-2",)- report_only = _index(- backend,+ report_only = index_document(title="C",content="reportage only here",- checksum="fuzz-kw-3",)return {"both": both.pk,@@ -182,7 +151,7 @@ class TestBooleanKeywordsInRawText:)def test_a_keyword_inside_a_phrase_stays_an_ordinary_word(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],corpus: dict[str, int],keyword_spelling: str,ordinary_spelling: str,@@ -201,14 +170,11 @@ class TestBooleanKeywordsInRawText:exclusion, IN does not fail the parse. Only the upper-casespelling was ever grammar"""- assert _matched_ids(backend, keyword_spelling) == _matched_ids(- backend,- ordinary_spelling,- )+ assert matched_ids(keyword_spelling) == matched_ids(ordinary_spelling)def test_a_phrase_needs_a_near_match_for_every_word(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],corpus: dict[str, int],) -> None:"""@@ -222,11 +188,11 @@ class TestBooleanKeywordsInRawText:quoted phrase asks for more than the bare words, so itsfuzzy side requires every one of them"""- assert _matched_ids(backend, '"tax reports"') == {corpus["both"]}+ assert matched_ids('"tax reports"') == {corpus["both"]}def test_a_trailing_keyword_is_just_a_word(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],corpus: dict[str, int],) -> None:"""@@ -242,7 +208,4 @@ class TestBooleanKeywordsInRawText:is re-parsed any more, so a trailing keyword cannot cost thequery its fuzzy side"""- assert _matched_ids(backend, '"tax AND"') == _matched_ids(- backend,- '"tax and"',- )+ assert matched_ids('"tax AND"') == matched_ids('"tax and"')diff --git a/src/documents/tests/search/test_highlight_query_guard.py b/src/documents/tests/search/test_highlight_query_guard.pyindex 48bf83502..cf9e96378 100644--- a/src/documents/tests/search/test_highlight_query_guard.py+++ b/src/documents/tests/search/test_highlight_query_guard.py@@ -29,8 +29,6 @@ from rest_framework import statusfrom documents.search._backend import SearchModefrom documents.search._query import parse_simple_text_highlight_query-from documents.search._schema import build_schema-from documents.search._tokenizer import register_tokenizersfrom documents.tests.factories import DocumentFactoryif TYPE_CHECKING:@@ -55,15 +53,6 @@ _MALFORMED_QUERIES = [][email protected](scope="module")-def query_index() -> tantivy.Index:- """An in-memory, unstemmed index for parse-only tests."""- schema = build_schema()- idx = tantivy.Index(schema, path=None)- register_tokenizers(idx, "")- return idx--class TestParseSimpleTextHighlightQueryDoesNotRaise:"""The query builder itself must tolerate Tantivy syntax in its tokens."""diff --git a/src/documents/tests/search/test_json_field_prefixes.py b/src/documents/tests/search/test_json_field_prefixes.pyindex 8d0ccedd4..df737f39d 100644--- a/src/documents/tests/search/test_json_field_prefixes.py+++ b/src/documents/tests/search/test_json_field_prefixes.py@@ -21,29 +21,24 @@ from django.contrib.auth.models import Userfrom documents.models import CustomFieldfrom documents.models import CustomFieldInstance-from documents.models import Documentfrom documents.models import Note+from documents.tests.factories import DocumentFactoryif TYPE_CHECKING:+ from collections.abc import Callable++ from documents.models import Documentfrom documents.search._backend import TantivyBackendpytestmark = [pytest.mark.search, pytest.mark.django_db]-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))---def _index(backend: TantivyBackend, **kwargs: object) -> Document:- doc = Document.objects.create(**kwargs)- backend.add_or_update(doc)- return doc--class TestBareJsonFieldPrefixes:def test_bare_notes_prefix_searches_note_text(self,backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -59,26 +54,22 @@ class TestBareJsonFieldPrefixes:text search"""alice = User.objects.create_user(username="alice")- with_note = Document.objects.create(- title="Has note",- content="x",- checksum="bare-notes-with",- )+ with_note = DocumentFactory(title="Has note", content="x")Note.objects.create(document=with_note, user=alice, note="crocodile")backend.add_or_update(with_note)# This document's CONTENT contains the words a demoted text search# would match; it must NOT match once the prefix addresses notes.- _index(- backend,+ index_document(title="Notes about things",content="notes crocodile mention",- checksum="bare-notes-decoy",)- assert _matched_ids(backend, "notes:crocodile") == {with_note.pk}+ assert matched_ids("notes:crocodile") == {with_note.pk}def test_bare_custom_fields_prefix_searches_values(self,backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -96,28 +87,23 @@ class TestBareJsonFieldPrefixes:name="Policy Number",data_type=CustomField.FieldDataType.STRING,)- with_value = Document.objects.create(- title="Has field",- content="x",- checksum="bare-cf-with",- )+ with_value = DocumentFactory(title="Has field", content="x")CustomFieldInstance.objects.create(document=with_value,field=field,value_text="crocodile",)backend.add_or_update(with_value)- _index(- backend,+ index_document(title="Custom things",content="custom fields crocodile",- checksum="bare-cf-decoy",)- assert _matched_ids(backend, "custom_fields:crocodile") == {with_value.pk}+ assert matched_ids("custom_fields:crocodile") == {with_value.pk}def test_subpath_spellings_are_untouched(self,backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -131,15 +117,11 @@ class TestBareJsonFieldPrefixes:prefix does not interfere with explicit subpath addressing"""bob = User.objects.create_user(username="bob")- doc = Document.objects.create(- title="Bob note",- content="x",- checksum="bare-subpath",- )+ doc = DocumentFactory(title="Bob note", content="x")Note.objects.create(document=doc, user=bob, note="remark")backend.add_or_update(doc)- assert _matched_ids(backend, "notes.user:bob") == {doc.pk}- assert _matched_ids(backend, "notes.note:remark") == {doc.pk}+ assert matched_ids("notes.user:bob") == {doc.pk}+ assert matched_ids("notes.note:remark") == {doc.pk}class TestQuotedPhraseContainingNotesColonIsNotCorrupted:@@ -152,7 +134,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:def test_quoted_phrase_with_notes_colon_matches_by_content(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -167,20 +150,16 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:that matches nothing (the bug the deleted regex rewritecaused, since it was blind to quoting)"""- target = _index(- backend,+ target = index_document(title="Statement",content="payment notes: none",- checksum="quoted-phrase-notes-colon",)- assert _matched_ids(- backend,- 'content:"payment notes: none"',- ) == {target.pk}+ assert matched_ids('content:"payment notes: none"') == {target.pk}def test_quoted_phrase_matches_the_same_document_unquoted(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],) -> None:"""GIVEN:@@ -194,13 +173,8 @@ class TestQuotedPhraseContainingNotesColonIsNotCorrupted:quote-awareness specifically, not about the wordsthemselves being unsearchable"""- target = _index(- backend,+ target = index_document(title="Statement",content="payment notes none",- checksum="quoted-phrase-no-colon",)- assert _matched_ids(- backend,- 'content:"payment notes none"',- ) == {target.pk}+ assert matched_ids('content:"payment notes none"') == {target.pk}diff --git a/src/documents/tests/search/test_negated_leaf_ids.py b/src/documents/tests/search/test_negated_leaf_ids.pyindex bea7069ec..9df5daec0 100644--- a/src/documents/tests/search/test_negated_leaf_ids.py+++ b/src/documents/tests/search/test_negated_leaf_ids.py@@ -11,19 +11,14 @@ identity.from __future__ import annotationsimport pytest-import whoosh_compat as wcimport whoosh_compat.ast as wc_astfrom documents.search._query import _negated_leaf_ids+from documents.tests.search._ast_helpers import CONTENT+from documents.tests.search._ast_helpers import contentpytestmark = pytest.mark.search-_CONTENT = wc.FieldRef("content")---def _term(text: str) -> wc_ast.Term:- return wc_ast.Term(field=_CONTENT, text=text)-class TestCollection:def test_a_bare_tree_has_no_negated_leaves(self) -> None:@@ -35,7 +30,7 @@ class TestCollection:THEN:- The set is empty"""- tree = wc_ast.And(children=(_term("invoice"), _term("report")))+ tree = wc_ast.And(children=(content("invoice"), content("report")))assert _negated_leaf_ids(tree) == frozenset()@@ -48,8 +43,8 @@ class TestCollection:THEN:- That leaf's id is collected, and the positive one is not"""- positive = _term("invoice")- negated = _term("secret")+ positive = content("invoice")+ negated = content("secret")tree = wc_ast.And(children=(positive, wc_ast.Not(child=negated)))ids = _negated_leaf_ids(tree)@@ -66,8 +61,8 @@ class TestCollection:THEN:- Only the negative side is collected"""- positive = _term("invoice")- negated = _term("secret")+ positive = content("invoice")+ negated = content("secret")tree = wc_ast.AndNot(positive=positive, negative=negated)ids = _negated_leaf_ids(tree)@@ -86,9 +81,9 @@ class TestCollection:- All of them are collected, at any depth and whatever theleaf type"""- a = _term("alpha")- b = _term("beta")- phrase = wc_ast.Phrase(field=_CONTENT, text="gamma delta")+ a = content("alpha")+ b = content("beta")+ phrase = wc_ast.Phrase(field=CONTENT, text="gamma delta")tree = wc_ast.Not(child=wc_ast.Or(children=(a, wc_ast.And(children=(b, phrase))),@@ -109,7 +104,7 @@ class TestCollection:under-collecting would make a negation exclude far more thanthe user asked"""- leaf = _term("tax")+ leaf = content("tax")tree = wc_ast.Not(child=wc_ast.Not(child=leaf))assert _negated_leaf_ids(tree) == frozenset({id(leaf)})@@ -144,7 +139,7 @@ class TestCollection:- Nothing is collected. Only Not.child and AndNot.negative arenegative positions"""- leaf = _term("invoice")+ leaf = content("invoice")assert _negated_leaf_ids(build(leaf)) == frozenset()@@ -157,7 +152,7 @@ class TestCollection:THEN:- The set is empty and nothing raises"""- assert _negated_leaf_ids(_term("invoice")) == frozenset()+ assert _negated_leaf_ids(content("invoice")) == frozenset()class TestTotality:@@ -167,11 +162,11 @@ class TestTotality:pytest.param(wc_ast.Every(), id="every"),pytest.param(wc_ast.Nothing(), id="nothing"),pytest.param(- wc_ast.Wildcard(field=_CONTENT, pattern="inv*"),+ wc_ast.Wildcard(field=CONTENT, pattern="inv*"),id="wildcard",),pytest.param(- wc_ast.Fuzzy(field=_CONTENT, text="invoce", distance=1, prefix=True),+ wc_ast.Fuzzy(field=CONTENT, text="invoce", distance=1, prefix=True),id="fuzzy",),],@@ -200,7 +195,7 @@ class TestTotality:- It completes. The walk is iterative, so depth costs heaprather than Python stack frames"""- leaf = _term("invoice")+ leaf = content("invoice")node: wc_ast.Node = leaffor _ in range(5000):node = wc_ast.Not(child=node)diff --git a/src/documents/tests/search/test_pattern_stemming.py b/src/documents/tests/search/test_pattern_stemming.pyindex 79bc4ff46..6fe4d8629 100644--- a/src/documents/tests/search/test_pattern_stemming.py+++ b/src/documents/tests/search/test_pattern_stemming.py@@ -18,10 +18,10 @@ from typing import TYPE_CHECKINGimport pytest-from documents.models import Document-if TYPE_CHECKING:- from documents.search._backend import TantivyBackend+ from collections.abc import Callable++ from documents.models import Documentpytestmark = [pytest.mark.search, pytest.mark.django_db]@@ -31,20 +31,13 @@ CONTENT = ()-def _matched_ids(backend: TantivyBackend, query: str) -> set[int]:- return set(backend.search_ids(query, user=None))--@pytest.fixture-def indexed_doc(backend: TantivyBackend) -> Document:- doc = Document.objects.create(+def indexed_doc(index_document: Callable[..., Document]) -> Document:+ return index_document(title="Invoice 2020 productname",content=CONTENT,- checksum="pattern-stemming-1",archive_serial_number=900,)- backend.add_or_update(doc)- return docclass TestPrefixStemming:@@ -61,7 +54,7 @@ class TestPrefixStemming:)def test_full_word_prefix_matches_its_stem(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,query: str,) -> None:@@ -78,12 +71,12 @@ class TestPrefixStemming:the word's stem as an alternative alongside the typed run,reaching the stemmed index term"""- assert _matched_ids(backend, query) == {indexed_doc.id}+ assert matched_ids(query) == {indexed_doc.id}@pytest.mark.parametrize("query", ["invoic*", "electr*", "payment*"])def test_already_stemmed_prefix_still_matches(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,query: str,) -> None:@@ -97,12 +90,12 @@ class TestPrefixStemming:- The document still matches, since the typed-run alternativeis itself a prefix of the stored stemmed term"""- assert _matched_ids(backend, query) == {indexed_doc.id}+ assert matched_ids(query) == {indexed_doc.id}@pytest.mark.parametrize("query", ["univers*", "librar*"])def test_partial_prefix_reaches_the_stemmed_term(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,query: str,) -> None:@@ -123,11 +116,11 @@ class TestPrefixStemming:genuinely diverge, and only one of them matches, istest_stem_substitution_reaches_both_the_inflection_and_the_compound"""- assert _matched_ids(backend, query) == {indexed_doc.id}+ assert matched_ids(query) == {indexed_doc.id}def test_full_word_reaches_the_stem_but_a_fragment_of_it_does_not(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,) -> None:"""@@ -144,13 +137,13 @@ class TestPrefixStemming:usage.md tells a reader whose `universit*` finds nothing toshorten it to `univers*`, which matches"""- assert _matched_ids(backend, "universities*") == {indexed_doc.id}- assert _matched_ids(backend, "universit*") == set()- assert _matched_ids(backend, "univers*") == {indexed_doc.id}+ assert matched_ids("universities*") == {indexed_doc.id}+ assert matched_ids("universit*") == set()+ assert matched_ids("univers*") == {indexed_doc.id}def test_pattern_past_the_stem_boundary_is_documented_not_fixed(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,) -> None:"""@@ -165,11 +158,12 @@ class TestPrefixStemming:index, and usage.md must not advertise it. Pinned so thelimitation is deliberate, not accidental"""- assert _matched_ids(backend, "produ*name") == set()+ assert matched_ids("produ*name") == set()def test_stem_substitution_reaches_both_the_inflection_and_the_compound(self,- backend: TantivyBackend,+ index_document: Callable[..., Document],+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,) -> None:"""@@ -189,22 +183,20 @@ class TestPrefixStemming:folded and stemmed forms, and "copy*" reaches the baseword, its inflections and the compound alike"""- compound = Document.objects.create(+ compound = index_document(title="Copyright notice",content="copyright notice for the work",- checksum="pattern-stemming-2",archive_serial_number=901,)- backend.add_or_update(compound)- assert _matched_ids(backend, "copy*") == {indexed_doc.id, compound.id}- assert _matched_ids(backend, "copyright*") == {compound.id}+ assert matched_ids("copy*") == {indexed_doc.id, compound.id}+ assert matched_ids("copyright*") == {compound.id}class TestBracketClassStillFolds:def test_class_body_matches_case_insensitively(self,- backend: TantivyBackend,+ matched_ids: Callable[[str], set[int]],indexed_doc: Document,) -> None:"""@@ -218,4 +210,4 @@ class TestBracketClassStillFolds:the alternatives contract preserves only because a lonecharacter stems to itself"""- assert _matched_ids(backend, "title:[IP]nvoice*") == {indexed_doc.id}+ assert matched_ids("title:[IP]nvoice*") == {indexed_doc.id}diff --git a/src/documents/tests/search/test_query.py b/src/documents/tests/search/test_query.pyindex aae802b85..4271df291 100644--- a/src/documents/tests/search/test_query.py+++ b/src/documents/tests/search/test_query.py@@ -24,16 +24,6 @@ if TYPE_CHECKING:pytestmark = pytest.mark.search[email protected](scope="module")-def query_index() -> tantivy.Index:- """An in-memory, unstemmed index shared read-only across this module's- parse-only tests (none of them index documents)."""- schema = build_schema()- idx = tantivy.Index(schema, path=None)- register_tokenizers(idx, "")- return idx--@pytest.fixture(scope="module")def populated_index() -> tantivy.Index:"""An index holding one document, so a query matching nothing isdiff --git a/src/documents/tests/search/test_query_negation.py b/src/documents/tests/search/test_query_negation.pyindex b967da2aa..d7090894b 100644--- a/src/documents/tests/search/test_query_negation.py+++ b/src/documents/tests/search/test_query_negation.py@@ -16,22 +16,12 @@ import pytestif TYPE_CHECKING:from collections.abc import Callable- from pytest_django.fixtures import SettingsWrapper-from documents.models import Documentpytestmark = [pytest.mark.search, pytest.mark.django_db]-def fuzzy_enabled(settings: SettingsWrapper) -> None:- """Enable the fuzzy blend clause. The threshold doubles as a minimum- score filter, so it is set to 0.0: every hit passes and the test sees- the clause's matching behaviour, not the filter's."""- settings.ADVANCED_FUZZY_SEARCH_THRESHOLD = 0.0--class TestNegationConstrainsEveryClause:@pytest.mark.usefixtures("fuzzy_enabled")def test_fuzzy_clause_does_not_readmit_an_excluded_document(diff --git a/src/documents/tests/search/test_widen_leaf.py b/src/documents/tests/search/test_widen_leaf.pyindex 258960c4d..c12a89ca4 100644--- a/src/documents/tests/search/test_widen_leaf.py+++ b/src/documents/tests/search/test_widen_leaf.py@@ -7,22 +7,16 @@ negated leaf keeps its CJK side while losing its fuzzy one.from __future__ import annotationsimport pytest-import whoosh_compat as wcimport whoosh_compat.ast as wc_astfrom documents.search._query import _cjk_alternativefrom documents.search._query import _fuzzy_alternativefrom documents.search._query import _widen_leaf+from documents.tests.search._ast_helpers import NOTES+from documents.tests.search._ast_helpers import contentpytestmark = pytest.mark.search-_CONTENT = wc.FieldRef("content")-_NOTES = wc.FieldRef("notes", "note")---def _content(text: str) -> wc_ast.Term:- return wc_ast.Term(field=_CONTENT, text=text)-def _widen(leaf: wc_ast.Term,@@ -44,7 +38,7 @@ class TestWhichAlternativesAreAdded:- The Or holds the leaf and the boosted fuzzy alternative,and nothing else"""- leaf = _content("invoice")+ leaf = content("invoice")assert _widen(leaf) == wc_ast.Or(children=(@@ -63,7 +57,7 @@ class TestWhichAlternativesAreAdded:- Only the bigram alternative is added, which is exactly whatthe CJK work shipped"""- leaf = _content("東京")+ leaf = content("東京")assert _widen(leaf, fuzzy=False) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)),@@ -81,7 +75,7 @@ class TestWhichAlternativesAreAdded:any run within one edit of its start, and _cjk_alternativealready supplies the in-run recall"""- leaf = _content("東京")+ leaf = content("東京")assert _fuzzy_alternative(leaf) is Noneassert _widen(leaf) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)))@@ -97,7 +91,7 @@ class TestWhichAlternativesAreAdded:NOT X should exclude what X matches, which needs the bigramside, but prefix fuzzy matching would exclude far more"""- leaf = _content("東京")+ leaf = content("東京")assert _widen(leaf, negated=frozenset({id(leaf)})) == wc_ast.Or(children=(leaf, _cjk_alternative(leaf)),@@ -113,7 +107,7 @@ class TestWhichAlternativesAreAdded:- The leaf itself comes back. It qualifies for no alternativeat all, so there is no Or to build"""- leaf = _content("tax")+ leaf = content("tax")assert _widen(leaf, negated=frozenset({id(leaf)})) is leaf@@ -128,6 +122,6 @@ class TestWhichAlternativesAreAdded:- The leaf comes back untouched. Widening is scoped to thedefault search fields, as it was for CJK"""- leaf = wc_ast.Term(field=_NOTES, text="invoice")+ leaf = wc_ast.Term(field=NOTES, text="invoice")assert _widen(leaf) is leafdiff --git a/src/documents/tests/test_api_search_errors.py b/src/documents/tests/test_api_search_errors.pyindex cd89d7355..c8a2849b9 100644--- a/src/documents/tests/test_api_search_errors.py+++ b/src/documents/tests/test_api_search_errors.py@@ -19,7 +19,6 @@ from whoosh_compat.errors import DiagnosticKindfrom whoosh_compat.errors import QueryErrorfrom documents.search import SearchQueryError-from documents.tests.factories import DocumentFactoryif TYPE_CHECKING:from rest_framework.test import APIClient@@ -29,15 +28,6 @@ if TYPE_CHECKING:pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]-def indexed_document() -> Document:- from documents.search import get_backend-- doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")- get_backend().add_or_update(doc)- return doc--class TestSearchQueryErrorStillBecomesA400:def test_search_query_error_becomes_a_400_naming_the_field(self,diff --git a/src/documents/tests/test_api_search_query_length.py b/src/documents/tests/test_api_search_query_length.pyindex 9ebc34aeb..2d52bc9ff 100644--- a/src/documents/tests/test_api_search_query_length.py+++ b/src/documents/tests/test_api_search_query_length.py@@ -22,7 +22,6 @@ import pytestfrom rest_framework import statusimport documents.search._backend-from documents.tests.factories import DocumentFactoryfrom documents.views import _MAX_QUERY_LENGTHif TYPE_CHECKING:@@ -33,15 +32,6 @@ if TYPE_CHECKING:pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]-def indexed_document() -> Document:- from documents.search import get_backend-- doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")- get_backend().add_or_update(doc)- return doc--class TestGetSearchEndpointEnforcesTheCap:def test_query_one_over_the_cap_is_a_400(self,diff --git a/src/documents/tests/test_api_search_unterminated_date_range.py b/src/documents/tests/test_api_search_unterminated_date_range.pyindex 0019de1b6..f8c26d743 100644--- a/src/documents/tests/test_api_search_unterminated_date_range.py+++ b/src/documents/tests/test_api_search_unterminated_date_range.py@@ -19,8 +19,6 @@ from typing import TYPE_CHECKINGimport pytestfrom rest_framework import status-from documents.tests.factories import DocumentFactory-if TYPE_CHECKING:from rest_framework.test import APIClient@@ -29,15 +27,6 @@ if TYPE_CHECKING:pytestmark = [pytest.mark.django_db, pytest.mark.usefixtures("_search_index")]-def indexed_document() -> Document:- from documents.search import get_backend-- doc = DocumentFactory.create(title="quarterly invoice", content="acme corp")- get_backend().add_or_update(doc)- return doc--class TestUnterminatedBracketReturnsA400:@pytest.mark.parametrize("query",