Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search: index title-only results (empty extract)

Daoud Clarke4d05b5047b8f
diff --git a/mwmbl/indexer/index_batches.py b/mwmbl/indexer/index_batches.py
index 89817a6..c952fa9 100644
--- a/mwmbl/indexer/index_batches.py
+++ b/mwmbl/indexer/index_batches.py
@@ -126,7 +126,7 @@ def index_results_against_query(documents: list[Document], query: str, index_pat
with TinyIndex(Document, index_path, 'r') as indexer:
existing_keys: dict[int, set[tuple]] = {}
for doc in documents:
- if not (doc.url and doc.title and doc.extract):
+ if not (doc.url and doc.title):
continue
doc_tokens = _document_token_set(doc)
for term, words in query_terms.items():
diff --git a/test/test_index_batches.py b/test/test_index_batches.py
index 013ea51..bc262b6 100644
--- a/test/test_index_batches.py
+++ b/test/test_index_batches.py
@@ -127,6 +127,27 @@ def test_index_results_against_query():
assert index_results_against_query(docs, "rust async", index_path) == 0
+def test_index_results_against_query_keeps_title_only_documents():
+ # Title-only results (empty extract) must still be indexed, matching what
+ # Super Search now keeps for display. The term is matched via the URL/title
+ # token set, so an empty extract should not exclude the document.
+ num_pages = 64
+ doc = Document(title="Kitsas dictionary", url="https://en.wiktionary.org/wiki/kitsas", extract="")
+ docs = [doc]
+
+ with TemporaryDirectory() as temp_dir:
+ index_path = str(Path(temp_dir) / 'temp-index.tinysearch')
+ with TinyIndex.create(Document, index_path, num_pages=num_pages, page_size=4096):
+ pass
+
+ new_count = index_results_against_query(docs, "kitsas", index_path)
+ assert new_count == 1
+
+ with TinyIndex(Document, index_path, 'r') as indexer:
+ kitsas_urls = {d.url for d in indexer.retrieve("kitsas")}
+ assert kitsas_urls == {doc.url}
+
+
# ---------------------------------------------------------------------------
# _merge_user_ids
# ---------------------------------------------------------------------------