An open source, non-profit web search engine
"""Tests for Super Search source selection: profiles, features, policy."""
import fakeredis
import numpy as np
import pytest
from mwmbl.tinysearchengine.indexer import Document
from mwmbl.tinysearchengine.super_search_select import (
features,
policy,
profiles,
rewards,
rstats,
vectors,
)
from mwmbl.tinysearchengine.super_search_select.features import QueryContext, SiteStats
from mwmbl.tinysearchengine.super_search_select.registry import SiteMeta
from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext, compute_rewards
DIM = 64
@pytest.fixture
def fake_profile_redis(monkeypatch):
"""Binary-safe fake Redis wired into the profiles and rstats modules."""
r = fakeredis.FakeRedis() # decode_responses=False (vectors are raw bytes)
monkeypatch.setattr(profiles, "_redis", r)
monkeypatch.setattr(rstats, "_redis", r)
return r
def _doc(title, extract=""):
return Document(title=title, url=f"https://x/{title}", extract=extract)
# ---------------------------------------------------------------------------
# Profiles
# ---------------------------------------------------------------------------
def test_unseen_profile_is_none(fake_profile_redis):
assert profiles.get_profile("nope") == (None, None)
def test_update_then_get_profile(fake_profile_redis):
profiles.update_profile("site", [_doc("python web framework", "django flask")])
bow, cng = profiles.get_profile("site")
assert bow is not None and cng is not None
assert abs(float(np.linalg.norm(bow)) - 1.0) < 1e-5
def test_profile_moves_towards_repeated_topic(fake_profile_redis):
# A site repeatedly returning ML content should profile closer to an ML query
# than to an unrelated one.
for _ in range(5):
profiles.update_profile("ml", [_doc("machine learning neural networks", "deep models")])
bow, _ = profiles.get_profile("ml")
ml_q = vectors.project_bow("neural network training", DIM)
other_q = vectors.project_bow("medieval french poetry", DIM)
assert vectors.cosine(ml_q, bow) > vectors.cosine(other_q, bow)
def test_empty_docs_no_profile(fake_profile_redis):
profiles.update_profile("site", [])
assert profiles.get_profile("site") == (None, None)
def test_query_vectors_cached(fake_profile_redis):
bow1, cng1 = profiles.get_query_vectors("hello world")
bow2, cng2 = profiles.get_query_vectors("hello world")
assert np.array_equal(bow1, bow2) and np.array_equal(cng1, cng2)
# Cached under the query key.
assert fake_profile_redis.exists(profiles._QVEC_BOW.format(key=vectors.query_cache_key("hello world")))
def test_batch_profiles(fake_profile_redis):
profiles.update_profile("a", [_doc("alpha")])
got = profiles.get_profiles(["a", "b"])
assert got["a"][0] is not None
assert got["b"] == (None, None)
# ---------------------------------------------------------------------------
# Features
# ---------------------------------------------------------------------------
def _qctx(query):
bow = vectors.project_bow(query, DIM)
cng = vectors.project_char_ngrams(query, DIM)
return QueryContext.build(query, bow, cng)
def test_feature_vector_shape_and_bias():
meta = SiteMeta("s", "s.com", "tech", popularity=0.6, estimated_pages=0.6)
x = features.feature_vector(_qctx("test query"), meta, (None, None))
assert x.shape == (features.NUM_FEATURES,)
assert x[0] == 1.0 # bias
def test_code_token_detected():
assert _qctx("os.path.join example").has_code_token
assert not _qctx("history of rome").has_code_token
def test_feature_vector_uses_stats():
meta = SiteMeta("s", "s.com", "tech")
stats = SiteStats(contribution_ema=0.5, latency_ema=10.0, failure_rate=0.25)
x = features.feature_vector(_qctx("q"), meta, (None, None), stats)
names = features.FEATURE_NAMES
assert x[names.index("contribution_ema")] == 0.5
assert x[names.index("latency_penalty")] == 1.0 # clamped (latency >> timeout)
assert x[names.index("failure_rate")] == 0.25
# ---------------------------------------------------------------------------
# Policy
# ---------------------------------------------------------------------------
def test_select_returns_all_when_fewer_than_k(fake_profile_redis):
names = ["a", "b", "c"]
assert set(policy.select_sources("q", names, k=10)) == set(names)
def test_select_includes_always_on(fake_profile_redis, monkeypatch):
# mwmbl and hn are always-on in the static registry.
names = ["mwmbl", "hn"] + [f"site{i}" for i in range(20)]
chosen = policy.select_sources("python testing", names, k=5)
assert "mwmbl" in chosen and "hn" in chosen
assert len(chosen) == 5
def test_select_respects_budget_and_records_features(fake_profile_redis):
# Selection runs the xgb policy against the bundled warm-start artifact.
names = ["db_site"] + [f"cold{i}" for i in range(20)]
ctx = SelectionContext()
chosen = policy.select_sources("sql database query", names, k=3, ctx=ctx)
assert len(chosen) == 3
assert set(chosen) <= set(ctx.features)
# ---------------------------------------------------------------------------
# Rewards
# ---------------------------------------------------------------------------
def test_compute_rewards_fraction_in_top_k():
ctx = SelectionContext(selected=["a", "b", "c"], per_source_limit=10)
ctx.record_results("a", ["u1", "u2", "u3"])
ctx.record_results("b", ["u4"])
# c returned nothing useful
final_top_k = ["u1", "u2", "u4"] # a:2, b:1, c:0
r = compute_rewards(ctx, final_top_k)
assert r["a"] == pytest.approx(0.2)
assert r["b"] == pytest.approx(0.1)
assert r["c"] == 0.0
def test_compute_rewards_clamped_and_covers_all_selected():
ctx = SelectionContext(selected=["a"], per_source_limit=2)
ctx.record_results("a", ["u1", "u2", "u3"])
r = compute_rewards(ctx, ["u1", "u2", "u3"]) # 3/2 -> clamp to 1.0
assert r == {"a": 1.0}
def test_record_results_first_source_wins():
ctx = SelectionContext(selected=["a", "b"], per_source_limit=10)
ctx.record_results("a", ["shared"])
ctx.record_results("b", ["shared"])
assert ctx.source_by_url["shared"] == "a"
def test_compute_judge_rewards_mean_per_source():
ctx = SelectionContext(selected=["a", "b", "c"])
ctx.record_results("a", ["u1", "u2"])
ctx.record_results("b", ["u3"])
# c returned nothing; u2 never reached the final pool so has no score
ctx.judge_scores = {"u1": 0.8, "u3": 0.4}
r = rewards.compute_judge_rewards(ctx)
assert r == {"a": pytest.approx(0.8), "b": pytest.approx(0.4), "c": 0.0}
def test_compute_judge_rewards_averages_multiple_docs():
ctx = SelectionContext(selected=["a"])
ctx.record_results("a", ["u1", "u2"])
ctx.judge_scores = {"u1": 1.0, "u2": 0.0}
assert rewards.compute_judge_rewards(ctx) == {"a": pytest.approx(0.5)}
def test_compute_judge_rewards_none_without_scores():
ctx = SelectionContext(selected=["a"])
ctx.record_results("a", ["u1"])
assert rewards.compute_judge_rewards(ctx) is None
def test_log_impression_noop_without_database(monkeypatch):
monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)
# Should not raise even though no DB is configured.
rewards.log_impression(SelectionContext(selected=["a"]), {"a": 0.5})
def test_record_source_provenance_noop_without_database(monkeypatch):
monkeypatch.setattr("django.conf.settings.HAS_DATABASE", False)
ctx = SelectionContext(selected=["a"])
ctx.record_results("a", ["https://x/u1"])
# Should not raise even though provenance is gated off.
rewards.record_source_provenance(ctx)
@pytest.mark.django_db
def test_record_source_provenance_writes_direct_rows(monkeypatch):
monkeypatch.setattr("django.conf.settings.HAS_DATABASE", True)
from mwmbl.models import SourceProvenance
ctx = SelectionContext(selected=["a", "b"])
ctx.record_results("a", ["https://x/u1", "https://x/u2"])
ctx.record_results("b", ["https://x/u3"])
rewards.record_source_provenance(ctx)
assert SourceProvenance.objects.count() == 3
row = SourceProvenance.objects.get(url="https://x/u1")
assert row.source == "a"
assert row.depth == 0
assert row.parent_url is None
# Idempotent: a second call must not raise or duplicate (unique url + ignore_conflicts).
rewards.record_source_provenance(ctx)
assert SourceProvenance.objects.count() == 3