Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Super Search: keep results that have no extract

Daoud Clarkecb08a2831ebb
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.py
index 4b145dc..0db5fa1 100644
--- a/mwmbl/tinysearchengine/super_search.py
+++ b/mwmbl/tinysearchengine/super_search.py
@@ -365,9 +365,9 @@ async def _follow_links(
raw_links = raw_links[:50]
await emit("page_fetched", PageFetchedEvent(url=parent.url, links=len(raw_links)))
- parent_title = content.get("title") or parent.title or _title_from_url(parent.url)
+ parent_title = content.get("title") or parent.title
parent_extract = content.get("extract") or parent.extract or ""
- if parent_title and parent_extract:
+ if parent_title:
all_docs.append(Document(title=parent_title, url=parent.url, extract=parent_extract))
if not raw_links:
@@ -394,13 +394,16 @@ async def _follow_links(
if not c:
continue
await emit("link_followed", LinkFollowedEvent(url=proxy_doc.url, **{"from": parent.url}))
- doc = Document(
- title=c.get("title") or _title_from_url(proxy_doc.url),
- url=proxy_doc.url,
- extract=c.get("extract") or "",
- )
- if doc.url and doc.title and doc.extract:
- all_docs.append(doc)
+ # Only keep links where the crawl produced a genuine page title; a
+ # URL-derived title (via _title_from_url) with no real content is a
+ # pseudo-result, not worth ranking or indexing. The extract may be empty.
+ crawled_title = (c.get("title") or "").strip()
+ if proxy_doc.url and crawled_title:
+ all_docs.append(Document(
+ title=crawled_title,
+ url=proxy_doc.url,
+ extract=c.get("extract") or "",
+ ))
await _emit_final_results(query, all_docs, emit, last_results_key, lock)
@@ -419,7 +422,7 @@ async def _emit_final_results(
seen: set[str] = set()
unique: list[Document] = []
for doc in all_docs:
- if doc.url and doc.title and doc.extract and doc.url not in seen:
+ if doc.url and doc.title and doc.url not in seen:
seen.add(doc.url)
unique.append(doc)
@@ -513,7 +516,7 @@ async def _run_pipeline(
scores = await asyncio.to_thread(_heuristic_score_docs, query, docs)
for doc, score in zip(docs, scores):
- if doc.url and doc.title and doc.extract:
+ if doc.url and doc.title:
all_docs.append(doc)
if _maybe_promote(doc, score):
await emit("result_promoted", _result_payload(doc, score, name, "direct"))
diff --git a/test/test_super_search.py b/test/test_super_search.py
index 473c166..917832f 100644
--- a/test/test_super_search.py
+++ b/test/test_super_search.py
@@ -472,6 +472,100 @@ def test_no_duplicate_consecutive_results_frames(client, api_key, monkeypatch):
assert prev != nxt, "consecutive identical results frames must be deduplicated"
+def test_title_only_source_result_retained(client, api_key, monkeypatch):
+ """A source/index result with an empty extract is kept (title + URL only),
+ matching standard Mwmbl search rather than being silently dropped.
+
+ Regression test for Super Search returning fewer results than standard search
+ (e.g. "olearia macrodonta", where two relevant pages have empty extracts).
+ """
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "mwmbl": [
+ Document(title="Python guide", url="https://py.example/", extract="A guide"),
+ Document(title="Python reference", url="https://noextract.example/", extract=""),
+ ],
+ })
+ _stub_scoring(monkeypatch, [0.9, 0.8] + [0.0] * 40)
+
+ def fake_crawl(url, redis):
+ return {"url": url, "status": 200, "content": None}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)
+ assert response.status_code == 200
+ events = _parse_sse(_read_stream(response))
+ final = [d for t, d in events if t == "results"][-1]["results"]
+ final_urls = {r["url"] for r in final}
+ assert "https://noextract.example/" in final_urls, "title-only result must not be dropped"
+
+
+@override_settings(SUPER_SEARCH_TOP_K=2)
+def test_followed_link_without_title_excluded(client, api_key, monkeypatch):
+ """A followed link whose crawl yields no real title is a pseudo-result
+ (its only title would be URL-derived) and must not enter the final ranking,
+ even though its extract matches the query."""
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "hn": [Document(title="Python parent", url="https://parent.example/",
+ extract="python parent text")],
+ })
+ _stub_scoring(monkeypatch, [0.9] + [0.0] * 20)
+
+ def fake_crawl(url, redis):
+ if url == "https://parent.example/":
+ return {"url": url, "status": 200, "content": {
+ "title": "Python parent", "extract": "python parent text",
+ "links": ["https://child.example/python-guide"], "extra_links": [],
+ }}
+ return {"url": url, "status": 200, "content": {
+ "title": "", "extract": "python child text", "links": [], "extra_links": [],
+ }}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)
+ assert response.status_code == 200
+ events = _parse_sse(_read_stream(response))
+ final = [d for t, d in events if t == "results"][-1]["results"]
+ final_urls = {r["url"] for r in final}
+ assert "https://child.example/python-guide" not in final_urls
+
+
+@override_settings(SUPER_SEARCH_TOP_K=2)
+def test_followed_link_with_title_and_no_extract_kept(client, api_key, monkeypatch):
+ """A followed link with a genuine crawled title but an empty extract is kept."""
+ cache.delete(_super_search_monthly_key(api_key.user.id))
+ _stub_sources(monkeypatch, {
+ "hn": [Document(title="Python parent", url="https://parent.example/",
+ extract="python parent text")],
+ })
+ _stub_scoring(monkeypatch, [0.9] + [0.0] * 20)
+
+ def fake_crawl(url, redis):
+ if url == "https://parent.example/":
+ return {"url": url, "status": 200, "content": {
+ "title": "Python parent", "extract": "python parent text",
+ "links": ["https://child.example/python-guide"], "extra_links": [],
+ }}
+ return {"url": url, "status": 200, "content": {
+ "title": "Python child", "extract": "", "links": [], "extra_links": [],
+ }}
+
+ monkeypatch.setattr("mwmbl.tinysearchengine.super_search.crawl_url", fake_crawl)
+
+ response = client.get("/api/v2/super-search/?q=python", HTTP_X_API_KEY=api_key.raw_key)
+ assert response.status_code == 200
+ events = _parse_sse(_read_stream(response))
+ final = [d for t, d in events if t == "results"][-1]["results"]
+ final_urls = {r["url"] for r in final}
+ assert "https://child.example/python-guide" in final_urls
+
+
# ---------------------------------------------------------------------------
# Architecture guard: no forbidden sync symbols in the orchestrator
# ---------------------------------------------------------------------------