Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Use whole-field heuristic score for super search promotion

Daoud Clarke4f03ea9dc29d
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 9d34811..1494bed 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -53,6 +53,26 @@ def score_result(terms: list[str], result: Document, is_complete: bool):
return 0.0
+def score_result_whole(terms: list[str], result: Document, is_complete: bool) -> float:
+ """Score a document using the combined whole-field match for promotion decisions.
+
+ Uses match_score_whole from get_features() without the majority-terms zero filter,
+ so partial matches score continuously rather than collapsing to 0.0.
+ """
+ if not result.url:
+ return 0.0
+ features = get_features(
+ terms,
+ result.title or '',
+ result.url,
+ result.extract or '',
+ result.score or 0.0,
+ is_complete,
+ )
+ length_penalty = math.e ** (-LENGTH_PENALTY * len(result.url))
+ return features['match_score_whole'] * length_penalty * (features['domain_score'] + DOMAIN_SCORE_SMOOTHING) / 10
+
+
def score_match(last_match_char, match_length, total_possible_match_length):
# return (match_length + 1. / last_match_char) / (total_possible_match_length + 1)
return MATCH_EXPONENT ** (match_length - total_possible_match_length) / last_match_char
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.py
index 497c8a3..b1990a9 100644
--- a/mwmbl/tinysearchengine/super_search.py
+++ b/mwmbl/tinysearchengine/super_search.py
@@ -34,6 +34,7 @@ from mwmbl.search_auth import SearchApiKeyAuth
from mwmbl.search_setup import ltr_model
from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.ltr_rank import score_documents
+from mwmbl.tinysearchengine.rank import score_result_whole
from mwmbl.tinysearchengine.super_search_sources import SOURCES
from mwmbl.tokenizer import tokenize
@@ -85,6 +86,13 @@ def _doc_passes_term_filter(doc: Document, terms: list[str]) -> bool:
return matches > len(terms) / 2
+def _heuristic_score_docs(query: str, docs: list[Document]) -> list[float]:
+ terms = tokenize(query)
+ if not terms:
+ return [0.0] * len(docs)
+ return [score_result_whole(terms, doc, is_complete=True) for doc in docs]
+
+
def _url_term_score(url: str, terms: list[str]) -> int:
"""Count how many query terms appear as whole words anywhere in the URL.
@@ -301,7 +309,7 @@ async def _run_pipeline(query: str, emit, all_docs: list[Document], last_results
if not docs:
continue
- scores = await asyncio.to_thread(score_documents, ltr_model, query, docs)
+ scores = await asyncio.to_thread(_heuristic_score_docs, query, docs)
for doc, score in zip(docs, scores):
if doc.url and doc.title and doc.extract:
all_docs.append(doc)
diff --git a/test/test_super_search.py b/test/test_super_search.py
index 2f88c88..918f704 100644
--- a/test/test_super_search.py
+++ b/test/test_super_search.py
@@ -94,15 +94,21 @@ def _stub_sources(monkeypatch, by_source: dict[str, list[Document]]):
def _stub_scoring(monkeypatch, scores: list[float]):
- """Make score_documents return the supplied list (one score per doc)."""
+ """Stub both the promotion scorer (_heuristic_score_docs) and the LTR final
+ ranker (score_documents) to consume from a single shared score iterator.
+ Promotion scores are consumed first; remaining scores go to final ranking."""
import mwmbl.tinysearchengine.super_search as ss
iterator = iter(scores)
- def fake_score(model, query, docs):
+ def fake_promote(query, docs):
return [next(iterator, 0.0) for _ in docs]
- monkeypatch.setattr(ss, "score_documents", fake_score)
+ def fake_ltr(model, query, docs):
+ return [next(iterator, 0.0) for _ in docs]
+
+ monkeypatch.setattr(ss, "_heuristic_score_docs", fake_promote)
+ monkeypatch.setattr(ss, "score_documents", fake_ltr)
# ---------------------------------------------------------------------------
@@ -196,6 +202,73 @@ def test_promoted_results_use_top_k(client, api_key, monkeypatch):
assert event_types[-1] == "done"
+@override_settings(SUPER_SEARCH_TOP_K=2)
+def test_heap_replacement_promotes_better_late_doc(client, api_key, monkeypatch):
+ """When the heap is full, a later doc with a higher score replaces the minimum and
+ is still promoted — the heap min-replacement path must work."""
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "hn": [
+ Document(title="Low A", url="https://low-a.example/", extract="x"),
+ Document(title="Low B", url="https://low-b.example/", extract="x"),
+ Document(title="Low C", url="https://low-c.example/", extract="x"),
+ Document(title="Best", url="https://best.example/", extract="x"),
+ ],
+ })
+ # First two fill the heap at 0.1; Low C can't enter (0.1 > 0.1 is False);
+ # Best (0.9) beats the minimum and must be promoted.
+ _stub_scoring(monkeypatch, [0.1, 0.1, 0.1, 0.9] + [0.0] * 20)
+
+ def fake_crawl(url):
+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get(
+ "/api/v2/super-search/?q=python",
+ HTTP_X_API_KEY=api_key.raw_key,
+ )
+ assert response.status_code == 200
+ events = _parse_sse(_read_stream(response))
+ promoted_urls = {d["url"] for t, d in events if t == "result_promoted"}
+
+ assert "https://best.example/" in promoted_urls, "High-score late doc must be promoted"
+ assert "https://low-c.example/" not in promoted_urls, "Equal-score doc must not displace heap entries"
+
+
+@override_settings(SUPER_SEARCH_TOP_K=2)
+def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch):
+ """A doc whose score equals the heap minimum must NOT be promoted — the check is
+ strictly greater-than, so ties don't displace existing entries."""
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "hn": [
+ Document(title="First", url="https://first.example/", extract="x"),
+ Document(title="Second", url="https://second.example/", extract="x"),
+ Document(title="Third", url="https://third.example/", extract="x"),
+ ],
+ })
+ _stub_scoring(monkeypatch, [0.5, 0.5, 0.5] + [0.0] * 20)
+
+ def fake_crawl(url):
+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get(
+ "/api/v2/super-search/?q=python",
+ HTTP_X_API_KEY=api_key.raw_key,
+ )
+ events = _parse_sse(_read_stream(response))
+ promoted = [d for t, d in events if t == "result_promoted"]
+ promoted_urls = {p["url"] for p in promoted}
+
+ assert len(promoted) == 2
+ assert "https://third.example/" not in promoted_urls
+
+
@pytest.mark.django_db
def test_final_results_event_emitted(client, api_key, monkeypatch):
"""A 'results' event with the full ranked list is emitted before 'done'."""