Skip to content

armgit / mwmbl

Public Imported from GitHub
0

An open source, non-profit web search engine

Pool Staan into the relabel dataset and evaluate Combined Search

Daoud Clarke5b3ffb7f4d17
diff --git a/mwmbl/rankeval/evaluation/compare_search_modes.py b/mwmbl/rankeval/evaluation/compare_search_modes.py
index a2ed3bb..71df606 100644
--- a/mwmbl/rankeval/evaluation/compare_search_modes.py
+++ b/mwmbl/rankeval/evaluation/compare_search_modes.py
@@ -1,7 +1,7 @@
"""
-Compare standard search vs Super Search v2 on the gold test set.
+Compare standard search, Super Search v2 and Combined Search on the gold test set.
-Both are scored against the same gold test split via ``rankeval.evaluation.evaluate``
+Every arm is scored against the same gold test split via ``rankeval.evaluation.evaluate``
on the same sampled queries (the evaluate module RNG is reseeded per model).
Fairness note — the standard-search baseline uses the *same* local mwmbl index,
@@ -12,6 +12,10 @@ PyPI, Stack Exchange, recipes), page crawling and outbound-link following. A
remote-index baseline (``evaluate_remote.py``) would be confounded by index size,
because Super Search's mwmbl source queries the local index, not the production one.
+Combined Search is the arm that is meant to replace Super Search: the same local index and
+LTR machinery, plus Staan and Wikipedia, with no crawling and no link-following. Its ablation
+arm pools the index and Wikipedia only, so the gap between the two is Staan's contribution.
+
Usage::
DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \
@@ -32,6 +36,7 @@ from django.conf import settings # noqa: E402
import mwmbl.rankeval.evaluation.evaluate as evaluate_module # noqa: E402
import mwmbl.rankeval.evaluation.evaluate_super_search as ss_module # noqa: E402
from mwmbl.rankeval.evaluation.evaluate import evaluate # noqa: E402
+from mwmbl.rankeval.evaluation.evaluate_combined_search import CombinedSearchRankingModel # noqa: E402
from mwmbl.rankeval.evaluation.evaluate_ranker import MwmblRankingModel # noqa: E402
from mwmbl.rankeval.evaluation.evaluate_super_search import SuperSearchRankingModel # noqa: E402
from mwmbl.search_setup import ranker # noqa: E402 (local mwmbl index + LTR + MMR + wiki)
@@ -63,6 +68,8 @@ def run():
("standard search (local index + wiki)", lambda: MwmblRankingModel(ranker)),
("super search v2 (cosine baseline, no new sources)", lambda: ss_arm([])),
("super search + new sources (gov.uk, imdb)", lambda: ss_arm(NEW_SOURCES)),
+ ("combined search (index + wiki, no staan)", lambda: CombinedSearchRankingModel(include_staan=False)),
+ ("combined search (index + staan + wiki)", lambda: CombinedSearchRankingModel()),
]
for label, make_model in models:
print(f"\n{'=' * 70}\nEvaluating {label}\n{'=' * 70}")
diff --git a/mwmbl/rankeval/evaluation/evaluate_combined_search.py b/mwmbl/rankeval/evaluation/evaluate_combined_search.py
new file mode 100644
index 0000000..71b3bc5
--- /dev/null
+++ b/mwmbl/rankeval/evaluation/evaluate_combined_search.py
@@ -0,0 +1,68 @@
+"""Evaluate the Combined Search ranking core as a RankingModel against the gold dataset.
+
+Runs what /api/v2/combined-search/ runs, minus the HTTP layer: pool the Mwmbl index, Staan
+and Wikipedia, rank the union with the combined LTR model, diversify. Only auth, quota and
+response formatting are skipped, so the ranking measured here is the ranking served.
+
+Not to be confused with evaluate_combined.py, which combines two ranking *models* (Wikipedia
+and Mwmbl) and predates this endpoint.
+
+Staan and Wikipedia are both network calls, so both are cached: Wikipedia through its own
+external results cache, and Staan through the same. A gold-set run therefore pays for each
+provider once per query and is cheap to repeat while iterating on the ranking. Pass
+``--no-staan`` to measure what Staan is actually contributing.
+
+Usage::
+
+ DJANGO_SETTINGS_MODULE=mwmbl.settings_dev \
+ uv run python -m mwmbl.rankeval.evaluation.evaluate_combined_search --fraction 0.05
+"""
+
+import os
+from argparse import ArgumentParser
+
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
+django.setup()
+
+# Imported after django.setup(): search_setup opens the index and loads the models.
+from mwmbl.rankeval.evaluation.evaluate import RankingModel, evaluate # noqa: E402
+from mwmbl.search_setup import combined_ranker # noqa: E402
+from mwmbl.tinysearchengine.rank import get_wiki_results # noqa: E402
+from mwmbl.tinysearchengine.staan import get_staan_results # noqa: E402
+
+
+class CombinedSearchRankingModel(RankingModel):
+ """The endpoint's ranking core: three sources pooled, one model, MMR.
+
+ include_staan=False is the ablation arm - the same pool minus Staan, so the difference
+ between the two arms is what Staan contributes and nothing else.
+ """
+
+ def __init__(self, ranker=combined_ranker, include_staan: bool = True):
+ self.ranker = ranker
+ self.include_staan = include_staan
+
+ def predict(self, query: str) -> list[str]:
+ additional = get_staan_results(query) if self.include_staan else []
+ additional = additional + get_wiki_results(query)
+ # use_external_search=False: Wikipedia is already in `additional`, exactly as the
+ # endpoint arranges it.
+ results = self.ranker.search(query, additional, False)
+ return [result.url for result in results]
+
+
+def run():
+ parser = ArgumentParser(description=__doc__)
+ parser.add_argument("--fraction", type=float, default=0.05, help="Fraction of gold queries to sample.")
+ parser.add_argument("--train", action="store_true", help="Evaluate on the train split instead of test.")
+ parser.add_argument("--no-staan", action="store_true", help="Ablation: pool the index and Wikipedia only.")
+ args = parser.parse_args()
+
+ model = CombinedSearchRankingModel(include_staan=not args.no_staan)
+ evaluate(model, fraction=args.fraction, use_test=not args.train)
+
+
+if __name__ == "__main__":
+ run()
diff --git a/scripts/_relabel_pool.py b/scripts/_relabel_pool.py
new file mode 100644
index 0000000..917d6d3
--- /dev/null
+++ b/scripts/_relabel_pool.py
@@ -0,0 +1,58 @@
+"""The Pass-2 candidate pool merge rule, shared by the collectors.
+
+Lives on its own so the rule cannot drift between the original three-pool collector and any
+later pass that adds a source to an already-collected pool: two copies of "keep the richest
+title, first score wins, accumulate provenance" would eventually stop agreeing, and the
+disagreement would show up as a feature shift in the training data rather than as an error.
+
+Kept free of django.setup() and of any ranker, so importing it costs nothing.
+"""
+
+
+def add(
+ pool: dict,
+ url: str,
+ title: str,
+ extract: str,
+ state,
+ score,
+ *,
+ pool_tag: str,
+ ss_source: str | None = None,
+ gold_rank: int | None = None,
+):
+ """Merge one result into the per-url pool, keeping the richest title/extract.
+
+ First writer wins for `score`, `state`, `ss_source` and `gold_rank`; `pools` accumulates.
+ That the first score wins matters: a URL two sources both returned keeps the score of
+ whichever was pooled first, which is how every row in the existing dataset was built.
+ """
+ if not url:
+ return
+ item = pool.setdefault(
+ url,
+ {
+ "url": url,
+ "title": "",
+ "extract": "",
+ "state": None,
+ "score": None,
+ "pools": [],
+ "ss_source": None,
+ "gold_rank": None,
+ },
+ )
+ if title and len(title) > len(item["title"]):
+ item["title"] = title
+ if extract and len(extract) > len(item["extract"]):
+ item["extract"] = extract
+ if score is not None and item["score"] is None:
+ item["score"] = float(score)
+ if state is not None and item["state"] is None:
+ item["state"] = state
+ if pool_tag not in item["pools"]:
+ item["pools"].append(pool_tag)
+ if ss_source and not item["ss_source"]:
+ item["ss_source"] = ss_source
+ if gold_rank is not None and item["gold_rank"] is None:
+ item["gold_rank"] = int(gold_rank)
diff --git a/scripts/llm_relabel_pass2_augment_staan.py b/scripts/llm_relabel_pass2_augment_staan.py
new file mode 100644
index 0000000..aa6795f
--- /dev/null
+++ b/scripts/llm_relabel_pass2_augment_staan.py
@@ -0,0 +1,160 @@
+"""LLM relabel — Pass 2 AUGMENT: add Staan's results to an already-collected pool.
+
+Combined Search pools the Mwmbl index, Staan and Wikipedia, so the model that ranks that
+union has to be trained on it. The existing pool already covers the index and Wikipedia (both
+arrive in the `standard` pool, via HeuristicAndWikiRanker); what it has never seen is Staan.
+
+Re-running llm_relabel_pass2_collect.py would redo the whole Super Search fan-out — crawls,
+link-following and all — for 849 queries to add one source. This adds only the Staan call,
+and merges it into the existing pool through the same rule the other pools were built with
+(scripts/_relabel_pool.add), so a Staan row is comparable with an existing one.
+
+Two steps, so a long network-bound run can never leave the pool half-written:
+
+1. ``--collect`` (default) appends one record per query to a checkpoint of its own. Resumable:
+ re-running skips queries already collected.
+2. ``--merge`` folds the checkpoint into pass2_pool.jsonl, writing via a temporary file and
+ renaming, so the pool is either the old one or the new one and never a partial one.
+
+The score written for a Staan result is staan_score(rank) — the same constant the serving
+path uses. `score` becomes the item_score feature, so a scale here that production never
+produces is a model trained on a feature it will not see. See STAAN_TOP_SCORE.
+
+Usage::
+ DJANGO_SETTINGS_MODULE=mwmbl.settings_dev DATABASE_URL="postgres://daoud@" \
+ uv run python scripts/llm_relabel_pass2_augment_staan.py --limit 5 # smoke
+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py # full run
+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py --status
+ ... uv run python scripts/llm_relabel_pass2_augment_staan.py --merge
+"""
+
+import json
+import os
+from argparse import ArgumentParser
+from collections import Counter
+
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "mwmbl.settings_dev")
+django.setup()
+
+from mwmbl.tinysearchengine.staan import get_staan_results # noqa: E402
+from scripts._relabel_pool import add # noqa: E402
+
+POOL = "devdata/llm_relabel/pass2_pool.jsonl"
+CHECKPOINT = "devdata/llm_relabel/pass2_staan.jsonl"
+POOL_TAG = "staan"
+
+
+def _read_jsonl(path: str) -> list[dict]:
+ if not os.path.exists(path):
+ return []
+ return [json.loads(line) for line in open(path) if line.strip()]
+
+
+def _pool_queries() -> list[str]:
+ return [record["query"] for record in _read_jsonl(POOL)]
+
+
+def _collected() -> set[str]:
+ return {record["query"] for record in _read_jsonl(CHECKPOINT)}
+
+
+def collect_query(query: str) -> dict:
+ """Staan's results for one query, in the shape the merge step expects."""
+ results = get_staan_results(query)
+ return {
+ "query": query,
+ "results": [
+ {"url": document.url, "title": document.title, "extract": document.extract, "score": document.score}
+ for document in results
+ ],
+ }
+
+
+def cmd_collect(limit: int | None):
+ todo = [query for query in _pool_queries() if query not in _collected()]
+ if limit:
+ todo = todo[:limit]
+ print(f"Collecting Staan results for {len(todo)} queries")
+
+ with open(CHECKPOINT, "a") as checkpoint:
+ for i, query in enumerate(todo, 1):
+ record = collect_query(query)
+ checkpoint.write(json.dumps(record) + "\n")
+ checkpoint.flush()
+ print(f"[{i}/{len(todo)}] {query!r}: {len(record['results'])} results")
+
+
+def cmd_merge():
+ collected = {record["query"]: record["results"] for record in _read_jsonl(CHECKPOINT)}
+ if not collected:
+ print(f"Nothing to merge: {CHECKPOINT} is empty")
+ return
+
+ added = 0
+ merged_into_existing = 0
+ records = _read_jsonl(POOL)
+ for record in records:
+ results = collected.get(record["query"])
+ if results is None:
+ continue
+ pool = {candidate["url"]: candidate for candidate in record["candidates"]}
+ for result in results:
+ existing = result["url"] in pool
+ add(
+ pool,
+ result["url"],
+ result["title"] or "",
+ result["extract"] or "",
+ None,
+ result["score"],
+ pool_tag=POOL_TAG,
+ ss_source=POOL_TAG,
+ )
+ if existing:
+ merged_into_existing += 1
+ else:
+ added += 1
+ record["candidates"] = list(pool.values())
+
+ temporary = POOL + ".tmp"
+ with open(temporary, "w") as out:
+ for record in records:
+ out.write(json.dumps(record) + "\n")
+ os.replace(temporary, POOL)
+
+ print(f"Merged {len(collected)} queries into {POOL}")
+ print(f" {added} new candidates, {merged_into_existing} already pooled by another source")
+
+
+def cmd_status():
+ queries = _pool_queries()
+ collected = _read_jsonl(CHECKPOINT)
+ print(f"collected {len(collected)} / {len(queries)} pool queries ({CHECKPOINT})")
+ if collected:
+ counts = [len(record["results"]) for record in collected]
+ print(f" {sum(counts)} Staan results, {sum(counts) / len(counts):.1f} avg/query")
+ print(f" queries with no Staan results: {sum(1 for c in counts if c == 0)}")
+
+ pools = Counter(tag for record in _read_jsonl(POOL) for c in record["candidates"] for tag in c["pools"])
+ print(f" pool membership in {POOL}: {dict(pools)}")
+
+
+def run():
+ parser = ArgumentParser(description=__doc__)
+ parser.add_argument("--limit", type=int, default=None, help="Collect at most this many queries.")
+ parser.add_argument("--merge", action="store_true", help="Fold the checkpoint into pass2_pool.jsonl.")
+ parser.add_argument("--status", action="store_true", help="Report progress and pool membership.")
+ args = parser.parse_args()
+
+ if args.status:
+ cmd_status()
+ elif args.merge:
+ cmd_merge()
+ else:
+ cmd_collect(args.limit)
+
+
+if __name__ == "__main__":
+ run()
diff --git a/scripts/llm_relabel_pass2_collect.py b/scripts/llm_relabel_pass2_collect.py
index 83ca464..4265e9c 100644
--- a/scripts/llm_relabel_pass2_collect.py
+++ b/scripts/llm_relabel_pass2_collect.py
@@ -45,6 +45,7 @@ from mwmbl.tinysearchengine.super_search import _run_pipeline # noqa: E402
from mwmbl.tinysearchengine.super_search_select.registry import get_meta # noqa: E402
from mwmbl.tinysearchengine.super_search_select.rewards import SelectionContext # noqa: E402
from mwmbl.tinysearchengine.super_search_sources import SOURCES # noqa: E402
+from scripts._relabel_pool import add # noqa: E402
PASS1 = "devdata/llm_relabel/pass1_intents.jsonl"
CHECKPOINT = "devdata/llm_relabel/pass2_pool.jsonl"
@@ -83,50 +84,6 @@ async def _collect_ss(query: str, sources: list[str]) -> tuple[list[Document], d
return all_docs, dict(ctx.source_by_url)
-def _add(
- pool: dict,
- url: str,
- title: str,
- extract: str,
- state,
- score,
- *,
- pool_tag: str,
- ss_source: str | None = None,
- gold_rank: int | None = None,
-):
- """Merge one result into the per-url pool, keeping the richest title/extract."""
- if not url:
- return
- item = pool.setdefault(
- url,
- {
- "url": url,
- "title": "",
- "extract": "",
- "state": None,
- "score": None,
- "pools": [],
- "ss_source": None,
- "gold_rank": None,
- },
- )
- if title and len(title) > len(item["title"]):
- item["title"] = title
- if extract and len(extract) > len(item["extract"]):
- item["extract"] = extract
- if score is not None and item["score"] is None:
- item["score"] = float(score)
- if state is not None and item["state"] is None:
- item["state"] = state
- if pool_tag not in item["pools"]:
- item["pools"].append(pool_tag)
- if ss_source and not item["ss_source"]:
- item["ss_source"] = ss_source
- if gold_rank is not None and item["gold_rank"] is None:
- item["gold_rank"] = int(gold_rank)
-
-
def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k: int) -> dict:
pool: dict = {}
gold_urls = set(gold["url"].tolist())
@@ -142,7 +99,7 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:
)
kept = std[:std_top_k] + [d for d in std[std_top_k:] if d.url in gold_urls]
for d in kept:
- _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")
+ add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="standard")
# super search (forced sources) + provenance. Drop results from always-on
# sources (e.g. hn) that Pass-1 did NOT choose: the pipeline pins them
@@ -154,11 +111,11 @@ def collect_query(query: str, sources: list[str], gold: pd.DataFrame, std_top_k:
src = source_by_url.get(d.url)
if src in stray:
continue
- _add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)
+ add(pool, d.url, d.title or "", d.extract or "", d.state, d.score, pool_tag="supersearch", ss_source=src)
# google gold (snippet only)
for _, row in gold.iterrows():
- _add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])
+ add(pool, row["url"], "", str(row.get("snippet") or ""), None, None, pool_tag="google", gold_rank=row["rank"])
return {"query": query, "sources": sources, "candidates": list(pool.values())}
diff --git a/test/test_relabel_pool_augment.py b/test/test_relabel_pool_augment.py
new file mode 100644
index 0000000..a894806
--- /dev/null
+++ b/test/test_relabel_pool_augment.py
@@ -0,0 +1,228 @@
+"""Tests for the Pass-2 pool merge rule and the Staan augmentation step.
+
+The merge rule decides what a training row's features are for a URL more than one source
+returned, so getting it wrong shifts features rather than raising - which is exactly the kind
+of mistake that only shows up as a model that underperforms for no visible reason.
+"""
+
+import json
+import sys
+from pathlib import Path
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+from scripts._relabel_pool import add # noqa: E402
+
+# ---------------------------------------------------------------------------
+# The merge rule
+# ---------------------------------------------------------------------------
+
+
+def test_a_new_url_enters_the_pool():
+ pool = {}
+
+ add(pool, "https://tokio.rs/", "Tokio", "An async runtime.", None, 6.0, pool_tag="staan", ss_source="staan")
+
+ assert pool["https://tokio.rs/"] == {
+ "url": "https://tokio.rs/",
+ "title": "Tokio",
+ "extract": "An async runtime.",
+ "state": None,
+ "score": 6.0,
+ "pools": ["staan"],
+ "ss_source": "staan",
+ "gold_rank": None,
+ }
+
+
+def test_a_url_with_no_url_is_ignored():
+ pool = {}
+
+ add(pool, "", "Tokio", "", None, 6.0, pool_tag="staan")
+
+ assert pool == {}
+
+
+def test_provenance_accumulates_across_sources():
+ """`pools` is what the dataset's calibration readout groups by, so a URL two sources both
+ returned has to say so rather than claim the first one only."""
+ pool = {}
+
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 2.0, pool_tag="standard")
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 6.0, pool_tag="staan", ss_source="staan")
+
+ assert pool["https://tokio.rs/"]["pools"] == ["standard", "staan"]
+
+
+def test_the_first_score_wins():
+ """A URL two sources both returned keeps the score of whichever was pooled first. Every
+ row in the existing dataset was built this way; changing it here would make the new Staan
+ rows incomparable with the ones already judged."""
+ pool = {}
+
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 2.0, pool_tag="standard")
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 6.0, pool_tag="staan")
+
+ assert pool["https://tokio.rs/"]["score"] == 2.0
+
+
+def test_the_richest_title_and_extract_win():
+ pool = {}
+
+ add(pool, "https://tokio.rs/", "Tokio", "Short.", None, 1.0, pool_tag="standard")
+ add(pool, "https://tokio.rs/", "Tokio - async Rust", "A much longer description.", None, 1.0, pool_tag="staan")
+
+ assert pool["https://tokio.rs/"]["title"] == "Tokio - async Rust"
+ assert pool["https://tokio.rs/"]["extract"] == "A much longer description."
+
+
+def test_a_repeated_pool_tag_is_not_duplicated():
+ pool = {}
+
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 1.0, pool_tag="staan")
+ add(pool, "https://tokio.rs/", "Tokio", "", None, 1.0, pool_tag="staan")
+
+ assert pool["https://tokio.rs/"]["pools"] == ["staan"]
+
+
+# ---------------------------------------------------------------------------
+# The augmentation step
+# ---------------------------------------------------------------------------
+
+
+def relabel_files(tmp_path, monkeypatch):
+ """Point the augment script at a throwaway pool and checkpoint."""
+ augment = pytest.importorskip("scripts.llm_relabel_pass2_augment_staan")
+
+ pool = tmp_path / "pass2_pool.jsonl"
+ checkpoint = tmp_path / "pass2_staan.jsonl"
+ monkeypatch.setattr(augment, "POOL", str(pool))
+ monkeypatch.setattr(augment, "CHECKPOINT", str(checkpoint))
+ return augment, pool, checkpoint
+
+
+def _write(path, records):
+ path.write_text("".join(json.dumps(record) + "\n" for record in records))
+
+
+def test_merge_adds_staan_candidates_to_the_pool(relabel_files):
+ augment, pool, checkpoint = relabel_files
+ _write(
+ pool,
+ [
+ {
+ "query": "tokio",
+ "sources": ["mwmbl"],
+ "candidates": [
+ {
+ "url": "https://blog.example.com/tokio",
+ "title": "A crawled page",
+ "extract": "",
+ "state": None,
+ "score": 1.0,
+ "pools": ["standard"],
+ "ss_source": None,
+ "gold_rank": None,
+ }
+ ],
+ }
+ ],
+ )
+ _write(
+ checkpoint,
+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],
+ )
+
+ augment.cmd_merge()
+
+ candidates = json.loads(pool.read_text().strip())["candidates"]
+ assert [c["url"] for c in candidates] == ["https://blog.example.com/tokio", "https://tokio.rs/"]
+ assert candidates[1]["pools"] == ["staan"]
+ assert candidates[1]["ss_source"] == "staan"
+ assert candidates[1]["score"] == 6.0
+
+
+def test_merge_folds_a_url_another_source_already_returned(relabel_files):
+ augment, pool, checkpoint = relabel_files
+ _write(
+ pool,
+ [
+ {
+ "query": "tokio",
+ "sources": ["mwmbl"],
+ "candidates": [
+ {
+ "url": "https://tokio.rs/",
+ "title": "Tokio",
+ "extract": "",
+ "state": None,
+ "score": 2.0,
+ "pools": ["standard"],
+ "ss_source": None,
+ "gold_rank": None,
+ }
+ ],
+ }
+ ],
+ )
+ _write(
+ checkpoint,
+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],
+ )
+
+ augment.cmd_merge()
+
+ candidates = json.loads(pool.read_text().strip())["candidates"]
+ assert len(candidates) == 1
+ assert candidates[0]["pools"] == ["standard", "staan"]
+ assert candidates[0]["score"] == 2.0
+
+
+def test_merge_leaves_queries_with_no_staan_record_alone(relabel_files):
+ augment, pool, checkpoint = relabel_files
+ original = {
+ "query": "untouched",
+ "sources": ["mwmbl"],
+ "candidates": [
+ {
+ "url": "https://example.com/",
+ "title": "T",
+ "extract": "",
+ "state": None,
+ "score": 1.0,
+ "pools": ["standard"],
+ "ss_source": None,
+ "gold_rank": None,
+ }
+ ],
+ }
+ _write(pool, [original])
+ _write(checkpoint, [{"query": "something else", "results": []}])
+
+ augment.cmd_merge()
+
+ assert json.loads(pool.read_text().strip()) == original
+
+
+def test_merge_is_idempotent(relabel_files):
+ """Re-running the merge must not double-count: the checkpoint stays on disk after a
+ merge, and the obvious mistake is to run it twice."""
+ augment, pool, checkpoint = relabel_files
+ _write(
+ pool,
+ [{"query": "tokio", "sources": [], "candidates": []}],
+ )
+ _write(
+ checkpoint,
+ [{"query": "tokio", "results": [{"url": "https://tokio.rs/", "title": "Tokio", "extract": "", "score": 6.0}]}],
+ )
+
+ augment.cmd_merge()
+ augment.cmd_merge()
+
+ candidates = json.loads(pool.read_text().strip())["candidates"]
+ assert [c["url"] for c in candidates] == ["https://tokio.rs/"]
+ assert candidates[0]["pools"] == ["staan"]