An open source, non-profit web search engine
Replace per-domain result filter with MMR diversity re-ranking
Daoud Clarke87dc9b24862c
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.pyindex 763a91f..d74111f 100644--- a/mwmbl/rankeval/evaluation/evaluate_remote.py+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py@@ -13,7 +13,7 @@ from mwmbl.rankeval.paths import MODEL_PATH, RUST_MODEL_PATHfrom mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.ltr import RustXGBPipelinefrom mwmbl.tinysearchengine.ltr_rank import LTRRanker-from mwmbl.tinysearchengine.rank import DomainLimitingRanker, Ranker, HeuristicRanker, HeuristicAndWikiRanker+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, HeuristicAndWikiRankerfrom mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate@@ -32,7 +32,7 @@ def run():# model = pickle.load(open(MODEL_PATH, 'rb'))model = RustXGBPipeline.from_model_path(str(RUST_MODEL_PATH))- ranker = DomainLimitingRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3)# ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter(), max_wiki_results=3)model = MwmblRankingModel(ranker)@@ -41,7 +41,7 @@ def run():def single_query(query: str):model = pickle.load(open(MODEL_PATH, 'rb'))- ranker = DomainLimitingRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3)# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())results = ranker.search(query, [])for result in results:diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.pyindex 9fb2922..0a5f409 100644--- a/mwmbl/search_setup.py+++ b/mwmbl/search_setup.py@@ -12,7 +12,7 @@ from mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import TinyIndex, Documentfrom mwmbl.tinysearchengine.ltr import RustXGBPipelinefrom mwmbl.tinysearchengine.ltr_rank import LTRRanker-from mwmbl.tinysearchengine.rank import DomainLimitingRanker, HeuristicAndWikiRanker+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRankerCURATED_DOMAINS_CACHE_KEY = "curated-domains"CURATED_DOMAINS_CACHE_TIMEOUT = 300@@ -34,6 +34,8 @@ tiny_index = TinyIndex(item_factory=Document, index_path=index_path)tiny_index.__enter__()ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))-ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True, num_wiki_results=3))+# Diversity (one-time-only-per-domain) is handled inside LTRRanker via MMR re-ranking,+# which demotes rather than drops same-domain results.+ranker = LTRRanker(tiny_index, completer, ltr_model, include_wiki=True, num_wiki_results=3)batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.pyindex 16dd1ab..ca2ebfe 100644--- a/mwmbl/tinysearchengine/ltr_rank.py+++ b/mwmbl/tinysearchengine/ltr_rank.py@@ -4,6 +4,10 @@ LTR (Learning-to-Rank) ranker that uses the Rust XGBoost pipeline for scoring.LTRRanker accepts any model with a sklearn-compatible predict(DataFrame) interface,including both the Python sklearn pipeline and the Rust RustXGBPipeline."""+import math+from collections import Counter+from urllib.parse import urlparse+import numpy as npfrom pandas import DataFramefrom sklearn.base import BaseEstimator@@ -11,6 +15,70 @@ from sklearn.base import BaseEstimatorfrom mwmbl.tinysearchengine.completer import Completerfrom mwmbl.tinysearchengine.indexer import Document, TinyIndexfrom mwmbl.tinysearchengine.rank import Ranker, get_wiki_results+from mwmbl.tokenizer import tokenize+++# Maximal Marginal Relevance (MMR) diversity parameters, applied in order_results.+# MMR re-orders the relevance-sorted candidates so near-duplicate / same-domain+# results are demoted rather than dropped (the big search engines diversify the+# list instead of hard-capping one result per domain).+MMR_LAMBDA = 0.7 # weight on relevance vs. diversity (1.0 = pure relevance, 0.0 = max diversity)+DOMAIN_SIMILARITY_WEIGHT = 0.8 # within the kernel: weight on same-domain vs. text overlap+++def _bag_of_words(doc: Document) -> Counter:+ return Counter(tokenize(f"{doc.title or ''} {doc.extract or ''}"))+++def _cosine(a: Counter, b: Counter) -> float:+ if not a or not b:+ return 0.0+ dot = sum(count * b[token] for token, count in a.items() if token in b)+ if dot == 0:+ return 0.0+ norm_a = math.sqrt(sum(c * c for c in a.values()))+ norm_b = math.sqrt(sum(c * c for c in b.values()))+ return dot / (norm_a * norm_b)+++def _similarity(bow_a: Counter, netloc_a: str, bow_b: Counter, netloc_b: str) -> float:+ """Domain-dominant similarity: same-domain pairs are penalised hardest, with a+ bag-of-words cosine as a secondary signal to catch cross-domain near-duplicates."""+ domain_sim = 1.0 if netloc_a and netloc_a == netloc_b else 0.0+ text_sim = _cosine(bow_a, bow_b)+ return DOMAIN_SIMILARITY_WEIGHT * domain_sim + (1 - DOMAIN_SIMILARITY_WEIGHT) * text_sim+++def mmr_rerank(ranked_pages: list[Document]) -> list[Document]:+ """Re-order a relevance-sorted list to demote near-duplicate / same-domain results.++ Greedy Maximal Marginal Relevance with the domain-dominant kernel above. Relevance+ is rank-based (scale-invariant): the i-th most relevant page has relevance+ (n - i) / n, so it does not depend on the model's compressed score magnitudes.+ Each candidate is discounted by its greatest similarity to an already-selected page,+ so e.g. the second result from a domain sinks below fresher domains but is never+ dropped. Complexity is O(n^2) over the (small) per-query candidate set.+ """+ n = len(ranked_pages)+ if n <= 2:+ return ranked_pages++ relevance = [(n - i) / n for i in range(n)]+ bows = [_bag_of_words(p) for p in ranked_pages]+ netlocs = [urlparse(p.url).netloc for p in ranked_pages]++ remaining = set(range(n))+ max_sim = [0.0] * n+ selected: list[int] = []+ while remaining:+ best = max(remaining, key=lambda i: MMR_LAMBDA * relevance[i] - (1 - MMR_LAMBDA) * max_sim[i])+ selected.append(best)+ remaining.discard(best)+ for j in remaining:+ sim = _similarity(bows[best], netlocs[best], bows[j], netlocs[j])+ if sim > max_sim[j]:+ max_sim[j] = sim+ return [ranked_pages[i] for i in selected]class LTRRanker(Ranker):@@ -71,10 +139,14 @@ class LTRRanker(Ranker):mask = predictions > 0.0filtered_predictions = predictions[mask]filtered_pages = np.array(results)[mask]+ if len(filtered_pages) == 0:+ return []- # Vectorized sorting+ # Sort by model relevance, then re-order with MMR to diversify the list+ # (demotes same-domain / near-duplicate results instead of dropping them).indices = np.argsort(filtered_predictions)[::-1]- return filtered_pages[indices].tolist()+ ranked_pages = filtered_pages[indices].tolist()+ return mmr_rerank(ranked_pages)def external_search(self, query: str) -> list[Document]:if self.include_wiki:diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.pyindex 1494bed..33ad103 100644--- a/mwmbl/tinysearchengine/rank.py+++ b/mwmbl/tinysearchengine/rank.py@@ -414,27 +414,3 @@ class HeuristicAndWikiRanker(HeuristicRanker):return []return results--class DomainLimitingRanker:- def __init__(self, ranker: Ranker):- self.ranker = ranker-- def search(self, s: str, additional_results: list[Document]):- results = self.ranker.search(s, additional_results)- seen_domains = set()-- filtered_results = []- for result in results:- domain = urlparse(result.url).netloc- if domain in seen_domains:- continue-- filtered_results.append(result)- seen_domains.add(domain)- return filtered_results-- def complete(self, q: str):- return self.ranker.complete(q)-- def get_raw_results(self, query: str):- return self.ranker.get_raw_results(query)diff --git a/test/test_ltr_rank.py b/test/test_ltr_rank.pynew file mode 100644index 0000000..5745197--- /dev/null+++ b/test/test_ltr_rank.py@@ -0,0 +1,45 @@+from mwmbl.tinysearchengine.indexer import Document+from mwmbl.tinysearchengine.ltr_rank import mmr_rerank+++def _doc(title, url, extract=""):+ return Document(title=title, url=url, extract=extract)+++def test_mmr_keeps_all_documents():+ # MMR demotes, it never drops: same-domain results must all survive.+ pages = [+ _doc("artoh/kitsas", "https://github.com/artoh/kitsas", "kitsas kirjanpito"),+ _doc("artoh/kitsasdocsy", "https://github.com/artoh/kitsasdocsy", "kitsas documentation"),+ _doc("artoh/kitsashealth", "https://github.com/artoh/kitsashealth", "health check for kitsas"),+ ]+ reranked = mmr_rerank(pages)+ assert {p.url for p in reranked} == {p.url for p in pages}+++def test_mmr_demotes_same_domain_below_fresh_domain():+ # Input is in relevance order: two github.com results then a lower-ranked+ # different-domain result. MMR should lift the fresh domain above the second+ # same-domain result, while keeping all three.+ a = _doc("Alpha repo", "https://github.com/x/alpha", "alpha project")+ b = _doc("Beta repo", "https://github.com/x/beta", "beta project")+ c = _doc("Gamma site", "https://example.org/gamma", "gamma encyclopedia entry")+ reranked = mmr_rerank([a, b, c])+ urls = [p.url for p in reranked]+ assert urls[0] == a.url # most relevant stays first+ assert urls.index(c.url) < urls.index(b.url) # fresh domain promoted above 2nd github+++def test_mmr_preserves_order_without_duplicates():+ # All distinct domains and content: relevance order should be untouched.+ pages = [+ _doc("First", "https://a.com/1", "apples"),+ _doc("Second", "https://b.com/2", "bananas"),+ _doc("Third", "https://c.com/3", "cherries"),+ ]+ assert [p.url for p in mmr_rerank(pages)] == [p.url for p in pages]+++def test_mmr_short_lists_are_unchanged():+ pages = [_doc("a", "https://github.com/x/a"), _doc("b", "https://github.com/x/b")]+ assert mmr_rerank(pages) == pages