An open source, non-profit web search engine
Super Search: keep results that have no extract
Daoud Clarkecb08a2831ebb
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.pyindex 4b145dc..0db5fa1 100644--- a/mwmbl/tinysearchengine/super_search.py+++ b/mwmbl/tinysearchengine/super_search.py@@ -365,9 +365,9 @@ async def _follow_links(raw_links = raw_links[:50]await emit("page_fetched", PageFetchedEvent(url=parent.url, links=len(raw_links)))- parent_title = content.get("title") or parent.title or _title_from_url(parent.url)+ parent_title = content.get("title") or parent.titleparent_extract = content.get("extract") or parent.extract or ""- if parent_title and parent_extract:+ if parent_title:all_docs.append(Document(title=parent_title, url=parent.url, extract=parent_extract))if not raw_links:@@ -394,13 +394,16 @@ async def _follow_links(if not c:continueawait emit("link_followed", LinkFollowedEvent(url=proxy_doc.url, **{"from": parent.url}))- doc = Document(- title=c.get("title") or _title_from_url(proxy_doc.url),- url=proxy_doc.url,- extract=c.get("extract") or "",- )- if doc.url and doc.title and doc.extract:- all_docs.append(doc)+ # Only keep links where the crawl produced a genuine page title; a+ # URL-derived title (via _title_from_url) with no real content is a+ # pseudo-result, not worth ranking or indexing. The extract may be empty.+ crawled_title = (c.get("title") or "").strip()+ if proxy_doc.url and crawled_title:+ all_docs.append(Document(+ title=crawled_title,+ url=proxy_doc.url,+ extract=c.get("extract") or "",+ ))await _emit_final_results(query, all_docs, emit, last_results_key, lock)@@ -419,7 +422,7 @@ async def _emit_final_results(seen: set[str] = set()unique: list[Document] = []for doc in all_docs:- if doc.url and doc.title and doc.extract and doc.url not in seen:+ if doc.url and doc.title and doc.url not in seen:seen.add(doc.url)unique.append(doc)@@ -513,7 +516,7 @@ async def _run_pipeline(scores = await asyncio.to_thread(_heuristic_score_docs, query, docs)for doc, score in zip(docs, scores):- if doc.url and doc.title and doc.extract:+ if doc.url and doc.title:all_docs.append(doc)if _maybe_promote(doc, score):await emit("result_promoted", _result_payload(doc, score, name, "direct"))diff --git a/test/test_super_search.py b/test/test_super_search.pyindex 473c166..917832f 100644--- a/test/test_super_search.py+++ b/test/test_super_search.py@@ -472,6 +472,100 @@ def test_no_duplicate_consecutive_results_frames(client, api_key, monkeypatch):assert prev != nxt, "consecutive identical results frames must be deduplicated"+def test_title_only_source_result_retained(client, api_key, monkeypatch):+ """A source/index result with an empty extract is kept (title + URL only),+ matching standard Mwmbl search rather than being silently dropped.++ Regression test for Super Search returning fewer results than standard search+ (e.g. "olearia macrodonta", where two relevant pages have empty extracts).+ """+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "mwmbl": [+ Document(title="Python guide", url="https://py.example/", extract="A guide"),+ Document(title="Python reference", url="https://noextract.example/", extract=""),+ ],+ })+ _stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)++ def fake_crawl(url, redis):+ return {"url": url, "status": 200, "content": None}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)+ assert response.status_code == 200+ events = _parse_sse(_read_stream(response))+ final = [d for t, d in events if t == "results"][-1]["results"]+ final_urls = {r["url"] for r in final}+ assert "https://noextract.example/" in final_urls, "title-only result must not be dropped"+++@override_settings(SUPER_SEARCH_TOP_K=2)+def test_followed_link_without_title_excluded(client, api_key, monkeypatch):+ """A followed link whose crawl yields no real title is a pseudo-result+ (its only title would be URL-derived) and must not enter the final ranking,+ even though its extract matches the query."""+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "hn": [Document(title="Python parent", url="https://parent.example/",+ extract="python parent text")],+ })+ _stub_scoring(monkeypatch, [0.9] + [0.0] * 20)++ def fake_crawl(url, redis):+ if url == "https://parent.example/":+ return {"url": url, "status": 200, "content": {+ "title": "Python parent", "extract": "python parent text",+ "links": ["https://child.example/python-guide"], "extra_links": [],+ }}+ return {"url": url, "status": 200, "content": {+ "title": "", "extract": "python child text", "links": [], "extra_links": [],+ }}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)+ assert response.status_code == 200+ events = _parse_sse(_read_stream(response))+ final = [d for t, d in events if t == "results"][-1]["results"]+ final_urls = {r["url"] for r in final}+ assert "https://child.example/python-guide" not in final_urls+++@override_settings(SUPER_SEARCH_TOP_K=2)+def test_followed_link_with_title_and_no_extract_kept(client, api_key, monkeypatch):+ """A followed link with a genuine crawled title but an empty extract is kept."""+ cache.delete(_super_search_monthly_key(api_key.user.id))+ _stub_sources(monkeypatch, {+ "hn": [Document(title="Python parent", url="https://parent.example/",+ extract="python parent text")],+ })+ _stub_scoring(monkeypatch, [0.9] + [0.0] * 20)++ def fake_crawl(url, redis):+ if url == "https://parent.example/":+ return {"url": url, "status": 200, "content": {+ "title": "Python parent", "extract": "python parent text",+ "links": ["https://child.example/python-guide"], "extra_links": [],+ }}+ return {"url": url, "status": 200, "content": {+ "title": "Python child", "extract": "", "links": [], "extra_links": [],+ }}++ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)++ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)+ assert response.status_code == 200+ events = _parse_sse(_read_stream(response))+ final = [d for t, d in events if t == "results"][-1]["results"]+ final_urls = {r["url"] for r in final}+ assert "https://child.example/python-guide" in final_urls++# ---------------------------------------------------------------------------# Architecture guard: no forbidden sync symbols in the orchestrator# ---------------------------------------------------------------------------