Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Replace per-domain result filter with MMR diversity re-ranking

Daoud Clarke87dc9b24862c
diff --git a/mwmbl/rankeval/evaluation/evaluate_remote.py b/mwmbl/rankeval/evaluation/evaluate_remote.py
index 763a91f..d74111f 100644
--- a/mwmbl/rankeval/evaluation/evaluate_remote.py
+++ b/mwmbl/rankeval/evaluation/evaluate_remote.py
@@ -13,7 +13,7 @@ from mwmbl.rankeval.paths import MODEL_PATH, RUST_MODEL_PATH
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.ltr import RustXGBPipeline
from mwmbl.tinysearchengine.ltr_rank import LTRRanker
-from mwmbl.tinysearchengine.rank import DomainLimitingRanker, Ranker, HeuristicRanker, HeuristicAndWikiRanker
+from mwmbl.tinysearchengine.rank import Ranker, HeuristicRanker, HeuristicAndWikiRanker
from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate
@@ -32,7 +32,7 @@ def run():
# model = pickle.load(open(MODEL_PATH, 'rb'))
model = RustXGBPipeline.from_model_path(str(RUST_MODEL_PATH))
- ranker = DomainLimitingRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))
+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3)
# ranker = HeuristicRanker(RemoteIndex(), DummyCompleter())
# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter(), max_wiki_results=3)
model = MwmblRankingModel(ranker)
@@ -41,7 +41,7 @@ def run():
def single_query(query: str):
model = pickle.load(open(MODEL_PATH, 'rb'))
- ranker = DomainLimitingRanker(LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3))
+ ranker = LTRRanker(RemoteIndex(), DummyCompleter(), model, True, 3)
# ranker = HeuristicAndWikiRanker(RemoteIndex(), DummyCompleter())
results = ranker.search(query, [])
for result in results:
diff --git a/mwmbl/search_setup.py b/mwmbl/search_setup.py
index 9fb2922..0a5f409 100644
--- a/mwmbl/search_setup.py
+++ b/mwmbl/search_setup.py
@@ -12,7 +12,7 @@ from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import TinyIndex, Document
from mwmbl.tinysearchengine.ltr import RustXGBPipeline
from mwmbl.tinysearchengine.ltr_rank import LTRRanker
-from mwmbl.tinysearchengine.rank import DomainLimitingRanker, HeuristicAndWikiRanker
+from mwmbl.tinysearchengine.rank import HeuristicAndWikiRanker
CURATED_DOMAINS_CACHE_KEY = "curated-domains"
CURATED_DOMAINS_CACHE_TIMEOUT = 300
@@ -34,6 +34,8 @@ tiny_index = TinyIndex(item_factory=Document, index_path=index_path)
tiny_index.__enter__()
ltr_model = RustXGBPipeline.from_model_path(str(settings.RUST_MODEL_PATH))
-ranker = DomainLimitingRanker(LTRRanker(tiny_index, completer, ltr_model, include_wiki=True, num_wiki_results=3))
+# Diversity (one-time-only-per-domain) is handled inside LTRRanker via MMR re-ranking,
+# which demotes rather than drops same-domain results.
+ranker = LTRRanker(tiny_index, completer, ltr_model, include_wiki=True, num_wiki_results=3)
batch_cache = BatchCache(Path(settings.DATA_PATH) / settings.BATCH_DIR_NAME)
diff --git a/mwmbl/tinysearchengine/ltr_rank.py b/mwmbl/tinysearchengine/ltr_rank.py
index 16dd1ab..ca2ebfe 100644
--- a/mwmbl/tinysearchengine/ltr_rank.py
+++ b/mwmbl/tinysearchengine/ltr_rank.py
@@ -4,6 +4,10 @@ LTR (Learning-to-Rank) ranker that uses the Rust XGBoost pipeline for scoring.
LTRRanker accepts any model with a sklearn-compatible predict(DataFrame) interface,
including both the Python sklearn pipeline and the Rust RustXGBPipeline.
"""
+import math
+from collections import Counter
+from urllib.parse import urlparse
+
import numpy as np
from pandas import DataFrame
from sklearn.base import BaseEstimator
@@ -11,6 +15,70 @@ from sklearn.base import BaseEstimator
from mwmbl.tinysearchengine.completer import Completer
from mwmbl.tinysearchengine.indexer import Document, TinyIndex
from mwmbl.tinysearchengine.rank import Ranker, get_wiki_results
+from mwmbl.tokenizer import tokenize
+
+
+# Maximal Marginal Relevance (MMR) diversity parameters, applied in order_results.
+# MMR re-orders the relevance-sorted candidates so near-duplicate / same-domain
+# results are demoted rather than dropped (the big search engines diversify the
+# list instead of hard-capping one result per domain).
+MMR_LAMBDA = 0.7 # weight on relevance vs. diversity (1.0 = pure relevance, 0.0 = max diversity)
+DOMAIN_SIMILARITY_WEIGHT = 0.8 # within the kernel: weight on same-domain vs. text overlap
+
+
+def _bag_of_words(doc: Document) -> Counter:
+ return Counter(tokenize(f"{doc.title or ''} {doc.extract or ''}"))
+
+
+def _cosine(a: Counter, b: Counter) -> float:
+ if not a or not b:
+ return 0.0
+ dot = sum(count * b[token] for token, count in a.items() if token in b)
+ if dot == 0:
+ return 0.0
+ norm_a = math.sqrt(sum(c * c for c in a.values()))
+ norm_b = math.sqrt(sum(c * c for c in b.values()))
+ return dot / (norm_a * norm_b)
+
+
+def _similarity(bow_a: Counter, netloc_a: str, bow_b: Counter, netloc_b: str) -> float:
+ """Domain-dominant similarity: same-domain pairs are penalised hardest, with a
+ bag-of-words cosine as a secondary signal to catch cross-domain near-duplicates."""
+ domain_sim = 1.0 if netloc_a and netloc_a == netloc_b else 0.0
+ text_sim = _cosine(bow_a, bow_b)
+ return DOMAIN_SIMILARITY_WEIGHT * domain_sim + (1 - DOMAIN_SIMILARITY_WEIGHT) * text_sim
+
+
+def mmr_rerank(ranked_pages: list[Document]) -> list[Document]:
+ """Re-order a relevance-sorted list to demote near-duplicate / same-domain results.
+
+ Greedy Maximal Marginal Relevance with the domain-dominant kernel above. Relevance
+ is rank-based (scale-invariant): the i-th most relevant page has relevance
+ (n - i) / n, so it does not depend on the model's compressed score magnitudes.
+ Each candidate is discounted by its greatest similarity to an already-selected page,
+ so e.g. the second result from a domain sinks below fresher domains but is never
+ dropped. Complexity is O(n^2) over the (small) per-query candidate set.
+ """
+ n = len(ranked_pages)
+ if n <= 2:
+ return ranked_pages
+
+ relevance = [(n - i) / n for i in range(n)]
+ bows = [_bag_of_words(p) for p in ranked_pages]
+ netlocs = [urlparse(p.url).netloc for p in ranked_pages]
+
+ remaining = set(range(n))
+ max_sim = [0.0] * n
+ selected: list[int] = []
+ while remaining:
+ best = max(remaining, key=lambda i: MMR_LAMBDA * relevance[i] - (1 - MMR_LAMBDA) * max_sim[i])
+ selected.append(best)
+ remaining.discard(best)
+ for j in remaining:
+ sim = _similarity(bows[best], netlocs[best], bows[j], netlocs[j])
+ if sim > max_sim[j]:
+ max_sim[j] = sim
+ return [ranked_pages[i] for i in selected]
class LTRRanker(Ranker):
@@ -71,10 +139,14 @@ class LTRRanker(Ranker):
mask = predictions > 0.0
filtered_predictions = predictions[mask]
filtered_pages = np.array(results)[mask]
+ if len(filtered_pages) == 0:
+ return []
- # Vectorized sorting
+ # Sort by model relevance, then re-order with MMR to diversify the list
+ # (demotes same-domain / near-duplicate results instead of dropping them).
indices = np.argsort(filtered_predictions)[::-1]
- return filtered_pages[indices].tolist()
+ ranked_pages = filtered_pages[indices].tolist()
+ return mmr_rerank(ranked_pages)
def external_search(self, query: str) -> list[Document]:
if self.include_wiki:
diff --git a/mwmbl/tinysearchengine/rank.py b/mwmbl/tinysearchengine/rank.py
index 1494bed..33ad103 100644
--- a/mwmbl/tinysearchengine/rank.py
+++ b/mwmbl/tinysearchengine/rank.py
@@ -414,27 +414,3 @@ class HeuristicAndWikiRanker(HeuristicRanker):
return []
return results
-
-class DomainLimitingRanker:
- def __init__(self, ranker: Ranker):
- self.ranker = ranker
-
- def search(self, s: str, additional_results: list[Document]):
- results = self.ranker.search(s, additional_results)
- seen_domains = set()
-
- filtered_results = []
- for result in results:
- domain = urlparse(result.url).netloc
- if domain in seen_domains:
- continue
-
- filtered_results.append(result)
- seen_domains.add(domain)
- return filtered_results
-
- def complete(self, q: str):
- return self.ranker.complete(q)
-
- def get_raw_results(self, query: str):
- return self.ranker.get_raw_results(query)
diff --git a/test/test_ltr_rank.py b/test/test_ltr_rank.py
new file mode 100644
index 0000000..5745197
--- /dev/null
+++ b/test/test_ltr_rank.py
@@ -0,0 +1,45 @@
+from mwmbl.tinysearchengine.indexer import Document
+from mwmbl.tinysearchengine.ltr_rank import mmr_rerank
+
+
+def _doc(title, url, extract=""):
+ return Document(title=title, url=url, extract=extract)
+
+
+def test_mmr_keeps_all_documents():
+ # MMR demotes, it never drops: same-domain results must all survive.
+ pages = [
+ _doc("artoh/kitsas", "https://github.com/artoh/kitsas", "kitsas kirjanpito"),
+ _doc("artoh/kitsasdocsy", "https://github.com/artoh/kitsasdocsy", "kitsas documentation"),
+ _doc("artoh/kitsashealth", "https://github.com/artoh/kitsashealth", "health check for kitsas"),
+ ]
+ reranked = mmr_rerank(pages)
+ assert {p.url for p in reranked} == {p.url for p in pages}
+
+
+def test_mmr_demotes_same_domain_below_fresh_domain():
+ # Input is in relevance order: two github.com results then a lower-ranked
+ # different-domain result. MMR should lift the fresh domain above the second
+ # same-domain result, while keeping all three.
+ a = _doc("Alpha repo", "https://github.com/x/alpha", "alpha project")
+ b = _doc("Beta repo", "https://github.com/x/beta", "beta project")
+ c = _doc("Gamma site", "https://example.org/gamma", "gamma encyclopedia entry")
+ reranked = mmr_rerank([a, b, c])
+ urls = [p.url for p in reranked]
+ assert urls[0] == a.url # most relevant stays first
+ assert urls.index(c.url) < urls.index(b.url) # fresh domain promoted above 2nd github
+
+
+def test_mmr_preserves_order_without_duplicates():
+ # All distinct domains and content: relevance order should be untouched.
+ pages = [
+ _doc("First", "https://a.com/1", "apples"),
+ _doc("Second", "https://b.com/2", "bananas"),
+ _doc("Third", "https://c.com/3", "cherries"),
+ ]
+ assert [p.url for p in mmr_rerank(pages)] == [p.url for p in pages]
+
+
+def test_mmr_short_lists_are_unchanged():
+ pages = [_doc("a", "https://github.com/x/a"), _doc("b", "https://github.com/x/b")]
+ assert mmr_rerank(pages) == pages