Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Use URL term overlap heuristic for outbound link selection in super search

Daoud Clarke3a0b26f2248a
diff --git a/mwmbl/tinysearchengine/super_search.py b/mwmbl/tinysearchengine/super_search.py
index f23a046..9151d41 100644
--- a/mwmbl/tinysearchengine/super_search.py
+++ b/mwmbl/tinysearchengine/super_search.py
@@ -88,6 +88,17 @@ def _doc_passes_term_filter(doc: Document, terms: list[str]) -> bool:
return features["match_terms"] > len(terms) / 2
+def _url_term_score(url: str, terms: list[str]) -> int:
+ """Count how many query terms appear anywhere in the URL (case-insensitive).
+
+ Used to rank outbound links before crawling them — the LTR model does poorly
+ with proxy docs that have only URL-derived titles, while simple term overlap
+ directly captures relevance signal available in the URL itself.
+ """
+ url_lower = url.lower()
+ return sum(1 for t in terms if t in url_lower)
+
+
def _title_from_url(url: str) -> str:
"""Cheap human-readable proxy used to score outbound links with the LTR model."""
try:
@@ -174,8 +185,9 @@ async def _follow_links(
if not raw_links:
return
+ terms = tokenize(query)
proxy_docs = [Document(title=_title_from_url(u), url=u, extract="") for u in raw_links]
- proxy_scores = await asyncio.to_thread(score_documents, ltr_model, query, proxy_docs)
+ proxy_scores = [_url_term_score(d.url, terms) for d in proxy_docs]
ranked = sorted(zip(proxy_docs, proxy_scores), key=lambda x: -x[1])[:max_links]
if not ranked: