An open source, non-profit web search engine
Use whole-field heuristic score for super search promotion
Daoud Clarke4f03ea9dc29d
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 9d34811..1494bed 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -53,6 +53,26 @@ def score_result(terms: list[str], result: Document, is_complete: bool):return 0.0+def score_result_whole(terms: list[str], result: Document, is_complete: bool) -> float:+ """Score a document using the combined whole-field match for promotion decisions.++ Uses match_score_whole from get_features() without the majority-terms zero filter,+ so partial matches score continuously rather than collapsing to 0.0.+ """+ if not result.url:+ return 0.0+ features = get_features(+ terms,+ result.title or '',+ result.url,+ result.extract or '',+ result.score or 0.0,+ is_complete,+ )+ length_penalty = math.e ** (-LENGTH_PENALTY * len(result.url))+ return features['match_score_whole'] * length_penalty * (features['domain_score'] + DOMAIN_SCORE_SMOOTHING) / 10++def score_match(last_match_char, match_length, total_possible_match_length):# return (match_length + 1. / last_match_char) / (total_possible_match_length + 1)return MATCH_EXPONENT ** (match_length - total_possible_match_length) / last_match_chardiff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 497c8a3..b1990a9 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -34,6 +34,7 @@ from mwmbl.search_auth import SearchApiKeyAuthfrom mwmbl.search_setup import ltr_modelfrom mwmbl.tinysearchengine.indexer import Documentfrom mwmbl.tinysearchengine.ltr_rank import score_documents+from mwmbl.tinysearchengine.rank import score_result_wholefrom mwmbl.tinysearchengine.super_search_sources import SOURCESfrom mwmbl.tokenizer import tokenize@@ -85,6 +86,13 @@ def _doc_passes_term_filter(doc: Document, terms: list[str]) -> bool:return matches > len(terms) / 2+def _heuristic_score_docs(query: str, docs: list[Document]) -> list[float]:+ terms = tokenize(query)+ if not terms:+ return [0.0] * len(docs)+ return [score_result_whole(terms, doc, is_complete=True) for doc in docs]++def _url_term_score(url: str, terms: list[str]) -> int:"""Count how many query terms appear as whole words anywhere in the URL.@@ -301,7 +309,7 @@ async def _run_pipeline(query: str, emit, all_docs: list[Document], last_resultsif not docs:continue- scores = await asyncio.to_thread(score_documents, ltr_model, query, docs)+ scores = await asyncio.to_thread(_heuristic_score_docs, query, docs)for doc, score in zip(docs, scores):if doc.url and doc.title and doc.extract:all_docs.append(doc)diff --git a/test/test_super_search.py b/test/test_super_search.pyindex 2f88c88..918f704 100644--- a/test/test_super_search.py+++ b/test/test_super_search.py@@ -94,15 +94,21 @@ def _stub_sources(monkeypatch, by_source: dict[str, list[Document]]):def _stub_scoring(monkeypatch, scores: list[float]):- """Make score_documents return the supplied list (one score per doc)."""+ """Stub both the promotion scorer (_heuristic_score_docs) and the LTR final+ ranker (score_documents) to consume from a single shared score iterator.+ Promotion scores are consumed first; remaining scores go to final ranking."""import mwmbl.tinysearchengine.super_search as ssiterator = iter(scores)- def fake_score(model, query, docs):+ def fake_promote(query, docs):return [next(iterator, 0.0) for _ in docs]- monkeypatch.setattr(ss, "score_documents", fake_score)+ def fake_ltr(model, query, docs):+ return [next(iterator, 0.0) for _ in docs]++ monkeypatch.setattr(ss, "_heuristic_score_docs", fake_promote)+ monkeypatch.setattr(ss, "score_documents", fake_ltr)# ---------------------------------------------------------------------------@@ -196,6 +202,73 @@ def test_promoted_results_use_top_k(client, api_key, monkeypatch):assert event_types[-1] == "done"+@override_settings(SUPER_SEARCH_TOP_K=2)+def test_heap_replacement_promotes_better_late_doc(client, api_key, monkeypatch):+ """When the heap is full, a later doc with a higher score replaces the minimum and+ is still promoted — the heap min-replacement path must work."""+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "hn": [+ Document(title="Low A", url="https://low-a.example/", extract="x"),+ Document(title="Low B", url="https://low-b.example/", extract="x"),+ Document(title="Low C", url="https://low-c.example/", extract="x"),+ Document(title="Best", url="https://best.example/", extract="x"),+ ],+ })+ # First two fill the heap at 0.1; Low C can't enter (0.1 > 0.1 is False);+ # Best (0.9) beats the minimum and must be promoted.+ _stub_scoring(monkeypatch, [0.1, 0.1, 0.1, 0.9] + [0.0] * 20)++ def fake_crawl(url):+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get(+ "/api/v2/super-search/?q=python",+ HTTP_X_API_KEY=api_key.raw_key,+ )+ assert response.status_code == 200+ events = _parse_sse(_read_stream(response))+ promoted_urls = {d["url"] for t, d in events if t == "result_promoted"}++ assert "https://best.example/" in promoted_urls, "High-score late doc must be promoted"+ assert "https://low-c.example/" not in promoted_urls, "Equal-score doc must not displace heap entries"+++@override_settings(SUPER_SEARCH_TOP_K=2)+def test_heap_equal_score_does_not_enter_full_heap(client, api_key, monkeypatch):+ """A doc whose score equals the heap minimum must NOT be promoted — the check is+ strictly greater-than, so ties don't displace existing entries."""+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "hn": [+ Document(title="First", url="https://first.example/", extract="x"),+ Document(title="Second", url="https://second.example/", extract="x"),+ Document(title="Third", url="https://third.example/", extract="x"),+ ],+ })+ _stub_scoring(monkeypatch, [0.5, 0.5, 0.5] + [0.0] * 20)++ def fake_crawl(url):+ return {"url": url, "status": 200, "timestamp": 0, "content": None, "error": None}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get(+ "/api/v2/super-search/?q=python",+ HTTP_X_API_KEY=api_key.raw_key,+ )+ events = _parse_sse(_read_stream(response))+ promoted = [d for t, d in events if t == "result_promoted"]+ promoted_urls = {p["url"] for p in promoted}++ assert len(promoted) == 2+ assert "https://third.example/" not in promoted_urls++@pytest.mark.django_dbdef test_final_results_event_emitted(client, api_key, monkeypatch):"""A 'results' event with the full ranked list is emitted before 'done'."""