Skip to content

"Vibe-Trading: Your Personal Trading Agent"

test(memory): add retrieval quality benchmark suite

shadowinlife6f9e304987cd
diff --git a/.gitignore b/.gitignore
index f83ca1eba..75e5adb14 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,3 +90,4 @@ AGENTS.md
# Alpha Zoo wiki artifacts (generated, not source)
wiki/alpha-library/manifest.json
wiki/alpha-library/content/
+bench_report.json
diff --git a/agent/tests/memory/benchmarks/__init__.py b/agent/tests/memory/benchmarks/__init__.py
new file mode 100644
index 000000000..c0ece3292
--- /dev/null
+++ b/agent/tests/memory/benchmarks/__init__.py
@@ -0,0 +1 @@
+"""P@5 evaluation framework for memory retrieval A/B testing."""
\ No newline at end of file
diff --git a/agent/tests/memory/benchmarks/conftest.py b/agent/tests/memory/benchmarks/conftest.py
new file mode 100644
index 000000000..08fd89776
--- /dev/null
+++ b/agent/tests/memory/benchmarks/conftest.py
@@ -0,0 +1,52 @@
+"""Shared fixtures for memory benchmark tests.
+
+Corpus files live outside version control (``tmp/benchmark_corpus/``).
+When they are absent the entire benchmark suite is skipped so CI stays green.
+The benchmark suite is designed to run locally after modifying the memory layer,
+not as part of the automated CI pipeline.
+"""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+# Project root (Vibe-Trading/)
+PROJECT_ROOT = Path(__file__).resolve().parents[4]
+CORPUS_DIR = PROJECT_ROOT / "tmp" / "benchmark_corpus"
+
+
+def _require_corpus() -> None:
+ """Skip the benchmark suite when corpus files are missing."""
+ memories = CORPUS_DIR / "memories_with_lifecycle.json"
+ queries = CORPUS_DIR / "queries.json"
+ if not memories.exists() or not queries.exists():
+ pytest.skip(
+ "Benchmark corpus files not found — run locally with corpus data "
+ "or regenerate via pipeline (see BENCHMARK_GUIDE.md §6).",
+ allow_module_level=True,
+ )
+
+
+# Fail-fast at collection time so the whole module is skipped when corpus
+# files are absent, rather than failing on every individual test.
+_require_corpus()
+
+
[email protected](scope="session")
+def memories_corpus() -> list[dict[str, Any]]:
+ """Load the 200-entry memory corpus with lifecycle metadata."""
+ path = CORPUS_DIR / "memories_with_lifecycle.json"
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
+
+
[email protected](scope="session")
+def queries_dataset() -> list[dict[str, Any]]:
+ """Load the 50-query evaluation dataset with ground-truth top-5."""
+ path = CORPUS_DIR / "queries.json"
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
\ No newline at end of file
diff --git a/agent/tests/memory/benchmarks/metrics.py b/agent/tests/memory/benchmarks/metrics.py
new file mode 100644
index 000000000..9f6929c99
--- /dev/null
+++ b/agent/tests/memory/benchmarks/metrics.py
@@ -0,0 +1,82 @@
+"""Retrieval quality metrics: P@K, MRR, NDCG@K.
+
+All functions follow the convention:
+ - `retrieved`: ordered list of memory IDs returned by the retrieval system
+ - `expected`: list of relevant memory IDs (ground truth)
+ - Higher is better for all metrics (range [0, 1])
+"""
+
+from __future__ import annotations
+
+import math
+
+
+def precision_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:
+ """P@K: fraction of top-K retrieved items that appear in ground truth.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+ k: Cut-off rank.
+
+ Returns:
+ Precision value in [0, 1].
+ """
+ if k <= 0 or not expected:
+ return 0.0
+ top_k = retrieved[:k]
+ relevant_set = set(expected)
+ hits = sum(1 for item in top_k if item in relevant_set)
+ return hits / k
+
+
+def mean_reciprocal_rank(retrieved: list[str], expected: list[str]) -> float:
+ """MRR: reciprocal of the rank of the first relevant result.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+
+ Returns:
+ Reciprocal rank in (0, 1] if a hit exists, else 0.0.
+ """
+ relevant_set = set(expected)
+ for rank, item in enumerate(retrieved, start=1):
+ if item in relevant_set:
+ return 1.0 / rank
+ return 0.0
+
+
+def ndcg_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:
+ """NDCG@K: Normalized Discounted Cumulative Gain at rank K.
+
+ Uses binary relevance (1 if in ground truth, 0 otherwise).
+ Position-aware: rewards relevant items appearing earlier.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+ k: Cut-off rank.
+
+ Returns:
+ NDCG value in [0, 1].
+ """
+ if k <= 0 or not expected:
+ return 0.0
+
+ relevant_set = set(expected)
+
+ # DCG for the actual retrieved ranking
+ dcg = 0.0
+ for i, item in enumerate(retrieved[:k]):
+ if item in relevant_set:
+ # Binary relevance: gain = 1
+ dcg += 1.0 / math.log2(i + 2) # i+2 because rank starts at 1
+
+ # Ideal DCG: all relevant items at the top
+ ideal_hits = min(len(relevant_set), k)
+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))
+
+ if idcg == 0.0:
+ return 0.0
+ return dcg / idcg
\ No newline at end of file
diff --git a/agent/tests/memory/benchmarks/runner.py b/agent/tests/memory/benchmarks/runner.py
new file mode 100644
index 000000000..5e727f54f
--- /dev/null
+++ b/agent/tests/memory/benchmarks/runner.py
@@ -0,0 +1,480 @@
+"""A/B comparison runner for memory retrieval evaluation.
+
+Simulates keyword-based retrieval with and without importance decay weighting,
+then computes P@5, MRR, NDCG@5 for both modes.
+
+Modes:
+ - Baseline (flag=0): pure token-overlap relevance scoring
+ - Treatment (flag=1): relevance × importance_weight (decay + quality)
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import random
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# 14-day half-life decay parameter
+HALF_LIFE_DAYS = 14.0
+DECAY_LAMBDA = math.log(2) / HALF_LIFE_DAYS
+
+# Access bonus coefficient (capped at 10 accesses)
+ACCESS_BONUS_COEFF = 0.1
+ACCESS_BONUS_CAP = 10
+
+# Metadata fields get higher weight in token matching
+METADATA_WEIGHT = 2.0
+
+# Top-K retrieval depth
+TOP_K = 5
+
+# CJK + Latin token regex
+_NON_LATIN_SCRIPT_RANGES = (
+ "一-鿿" # CJK Unified Ideographs
+ "㐀-䶿" # CJK Extension A
+)
+_LATIN_TOKEN_RE = re.compile(r"[a-zA-Z0-9]{3,}")
+_CJK_CHAR_RE = re.compile(rf"[{_NON_LATIN_SCRIPT_RANGES}]")
+
+
+# ---------------------------------------------------------------------------
+# Data structures
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MemoryRecord:
+ """In-memory representation of a benchmark memory entry."""
+
+ id: str
+ name: str
+ description: str
+ content: str
+ keywords: list[str]
+ quality_score: float
+ access_count: int
+ last_accessed_days_ago: float
+ created_days_ago: float
+
+ # Pre-computed token sets for fast matching
+ meta_tokens: set[str] = field(default_factory=set, repr=False)
+ keyword_tokens: set[str] = field(default_factory=set, repr=False)
+ body_tokens: set[str] = field(default_factory=set, repr=False)
+
+
+@dataclass
+class QueryRecord:
+ """In-memory representation of a benchmark query."""
+
+ id: str
+ query: str
+ difficulty: str
+ ground_truth_top5: list[str]
+ ranking_depends_on: str | None
+ category: str
+
+
+@dataclass
+class ABResult:
+ """Aggregated A/B comparison results."""
+
+ corpus_size: int
+ query_count: int
+ baseline_p5: float
+ baseline_mrr: float
+ baseline_ndcg5: float
+ treatment_p5: float
+ treatment_mrr: float
+ treatment_ndcg5: float
+ by_difficulty: dict[str, dict[str, float]]
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer
+# ---------------------------------------------------------------------------
+
+
+def tokenize_baseline(text: str) -> set[str]:
+ """Tokenization: Latin words (>=3 chars) + individual CJK characters."""
+ tokens: set[str] = set()
+ tokens.update(_LATIN_TOKEN_RE.findall(text.lower()))
+ tokens.update(_CJK_CHAR_RE.findall(text))
+ return tokens
+
+
+# Use same tokenization for both modes (the difference is in scoring)
+tokenize = tokenize_baseline
+
+
+# ---------------------------------------------------------------------------
+# Scoring functions
+# ---------------------------------------------------------------------------
+
+
+def compute_relevance(query_tokens: set[str], record: MemoryRecord) -> float:
+ """Token-overlap relevance score (uniform weight, for baseline)."""
+ meta_overlap = len(query_tokens & record.meta_tokens) * METADATA_WEIGHT
+ kw_overlap = len(query_tokens & record.keyword_tokens) * METADATA_WEIGHT
+ body_overlap = len(query_tokens & record.body_tokens)
+ return meta_overlap + kw_overlap + body_overlap
+
+
+def compute_relevance_bm25(
+ query_tokens: set[str],
+ record: MemoryRecord,
+ idf: dict[str, float],
+ avg_doc_len: float,
+) -> float:
+ """BM25-style relevance scoring (for treatment).
+
+ Improvements over uniform baseline:
+ 1. IDF weighting: rare/discriminative tokens (stock codes, names) score higher
+ 2. Length normalization: shorter focused entries aren't penalized vs long ones
+ 3. Term saturation: prevents single-token dominance
+
+ Parameters:
+ k1 = 1.2 (term frequency saturation)
+ b = 0.75 (length normalization strength)
+ """
+ k1 = 1.2
+ b = 0.75
+
+ doc_len = len(record.meta_tokens | record.keyword_tokens | record.body_tokens)
+ norm = 1.0 - b + b * (doc_len / avg_doc_len)
+
+ score = 0.0
+ # Binary TF (token present = 1)
+ tf = 1.0
+ tf_component = (tf * (k1 + 1.0)) / (tf + k1 * norm)
+
+ for token in query_tokens & record.meta_tokens:
+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT
+ for token in query_tokens & record.keyword_tokens:
+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT
+ for token in query_tokens & record.body_tokens:
+ score += idf.get(token, 1.0) * tf_component
+ return score
+
+
+def compute_importance_weight(record: MemoryRecord) -> float:
+ """Importance weight combining quality, decay, and access frequency.
+
+ Formula (mirrors production `compute_importance` in persistent.py):
+ raw = quality_score × (exp(-λ × days_ago) + access_bonus)
+ importance = clamp(raw, 0.0, 1.0)
+
+ Then used as: final_score = relevance × (0.98 + 0.02 × importance)
+ This ensures importance provides a controlled boost [0.98x, 1.0x] on
+ top of relevance, matching the production `find_relevant` behavior.
+
+ Parameters:
+ λ = ln(2) / 14 (14-day half-life)
+ access_bonus = 0.1 × min(access_count, 10)
+ """
+ retention = math.exp(-DECAY_LAMBDA * max(0.0, record.last_accessed_days_ago))
+ access_bonus = ACCESS_BONUS_COEFF * min(record.access_count, ACCESS_BONUS_CAP)
+ raw = record.quality_score * (retention + access_bonus)
+ return min(1.0, max(0.0, raw)) # Clamped to [0, 1] per production logic
+
+
+# ---------------------------------------------------------------------------
+# Retrieval simulation
+# ---------------------------------------------------------------------------
+
+
+def retrieve_top_k(
+ query_tokens: set[str],
+ corpus: list[MemoryRecord],
+ treatment: bool = False,
+ k: int = TOP_K,
+ idf: dict[str, float] | None = None,
+ avg_doc_len: float = 1.0,
+) -> list[str]:
+ """Retrieve top-K memory IDs by scoring.
+
+ Args:
+ query_tokens: Tokenized query.
+ corpus: All memory records.
+ treatment: If True, use BM25-style scoring + importance boost.
+ k: Number of results to return.
+ idf: Token IDF scores (required when treatment=True).
+ avg_doc_len: Average document length (required when treatment=True).
+
+ Returns:
+ Ordered list of memory IDs (best first).
+ """
+ scored: list[tuple[float, str]] = []
+
+ for record in corpus:
+ if treatment and idf is not None:
+ relevance = compute_relevance_bm25(
+ query_tokens, record, idf, avg_doc_len
+ )
+ else:
+ relevance = compute_relevance(query_tokens, record)
+
+ if relevance <= 0:
+ continue
+
+ if treatment:
+ importance = compute_importance_weight(record)
+ # Importance as tiebreaker: BM25 dominates, importance only
+ # affects entries with very similar relevance scores.
+ final_score = relevance * (0.98 + 0.02 * importance)
+ else:
+ final_score = relevance
+
+ scored.append((final_score, record.id))
+
+ # Sort by score descending, then by ID for deterministic tie-breaking
+ scored.sort(key=lambda x: (-x[0], x[1]))
+ return [mem_id for _, mem_id in scored[:k]]
+
+
+# ---------------------------------------------------------------------------
+# Corpus loading
+# ---------------------------------------------------------------------------
+
+
+def load_corpus(data: list[dict[str, Any]]) -> tuple[list[MemoryRecord], dict[str, float], float]:
+ """Convert raw JSON corpus to MemoryRecord list, IDF dict, and avg doc length.
+
+ Returns:
+ Tuple of (records, idf_dict, avg_doc_len).
+ """
+ records: list[MemoryRecord] = []
+ # First pass: create records and tokenize
+ for entry in data:
+ lifecycle = entry.get("lifecycle", {})
+ record = MemoryRecord(
+ id=entry["id"],
+ name=entry.get("name", ""),
+ description=entry.get("description", ""),
+ content=entry.get("content", ""),
+ keywords=entry.get("keywords", []),
+ quality_score=lifecycle.get("quality_score", 0.5),
+ access_count=lifecycle.get("access_count", 0),
+ last_accessed_days_ago=lifecycle.get("last_accessed_days_ago", 0.0),
+ created_days_ago=lifecycle.get("created_days_ago", 0.0),
+ )
+ meta_text = f"{record.name} {record.description}"
+ kw_text = " ".join(record.keywords)
+ record.meta_tokens = tokenize(meta_text)
+ record.keyword_tokens = tokenize(kw_text)
+ record.body_tokens = tokenize(record.content)
+ records.append(record)
+
+ # Second pass: compute IDF (Inverse Document Frequency)
+ n_docs = len(records)
+ doc_freq: dict[str, int] = {} # token -> number of docs containing it
+ doc_lengths: list[int] = []
+ for record in records:
+ all_tokens = record.meta_tokens | record.keyword_tokens | record.body_tokens
+ doc_lengths.append(len(all_tokens))
+ for token in all_tokens:
+ doc_freq[token] = doc_freq.get(token, 0) + 1
+
+ # IDF = log(N / df) with smoothing
+ idf: dict[str, float] = {}
+ for token, df in doc_freq.items():
+ idf[token] = math.log((n_docs + 1) / (df + 1)) + 1.0 # Smoothed IDF
+
+ avg_doc_len = sum(doc_lengths) / len(doc_lengths) if doc_lengths else 1.0
+
+ return records, idf, avg_doc_len
+
+
+def load_queries(data: list[dict[str, Any]]) -> list[QueryRecord]:
+ """Convert raw JSON queries to QueryRecord list."""
+ return [
+ QueryRecord(
+ id=entry["id"],
+ query=entry["query"],
+ difficulty=entry["difficulty"],
+ ground_truth_top5=entry["ground_truth_top5"],
+ ranking_depends_on=entry.get("ranking_depends_on"),
+ category=entry.get("category", ""),
+ )
+ for entry in data
+ ]
+
+
+# ---------------------------------------------------------------------------
+# A/B comparison runner
+# ---------------------------------------------------------------------------
+
+
+def run_ab_comparison(
+ corpus_data: list[dict[str, Any]],
+ queries_data: list[dict[str, Any]],
+) -> ABResult:
+ """Run full A/B comparison: baseline vs treatment.
+
+ Args:
+ corpus_data: Raw memory corpus JSON.
+ queries_data: Raw queries JSON.
+
+ Returns:
+ ABResult with all metrics.
+ """
+ random.seed(42)
+
+ corpus, idf, avg_doc_len = load_corpus(corpus_data)
+ queries = load_queries(queries_data)
+
+ # Per-query metrics
+ baseline_p5_scores: list[float] = []
+ baseline_mrr_scores: list[float] = []
+ baseline_ndcg5_scores: list[float] = []
+ treatment_p5_scores: list[float] = []
+ treatment_mrr_scores: list[float] = []
+ treatment_ndcg5_scores: list[float] = []
+
+ # By-difficulty tracking
+ difficulty_scores: dict[str, dict[str, list[float]]] = {
+ "easy": {"baseline_p5": [], "treatment_p5": []},
+ "medium": {"baseline_p5": [], "treatment_p5": []},
+ "hard": {"baseline_p5": [], "treatment_p5": []},
+ }
+
+ for q in queries:
+ query_tokens = tokenize(q.query)
+
+ # Baseline retrieval (uniform token weights, no importance)
+ baseline_results = retrieve_top_k(
+ query_tokens, corpus, treatment=False
+ )
+ baseline_p5_scores.append(
+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+ baseline_mrr_scores.append(
+ mean_reciprocal_rank(baseline_results, q.ground_truth_top5)
+ )
+ baseline_ndcg5_scores.append(
+ ndcg_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Treatment retrieval (BM25-weighted + importance boost)
+ treatment_results = retrieve_top_k(
+ query_tokens, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ treatment_p5_scores.append(
+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+ treatment_mrr_scores.append(
+ mean_reciprocal_rank(treatment_results, q.ground_truth_top5)
+ )
+ treatment_ndcg5_scores.append(
+ ndcg_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Track by difficulty
+ diff = q.difficulty
+ if diff in difficulty_scores:
+ difficulty_scores[diff]["baseline_p5"].append(
+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+ difficulty_scores[diff]["treatment_p5"].append(
+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Aggregate
+ def mean(values: list[float]) -> float:
+ return sum(values) / len(values) if values else 0.0
+
+ by_difficulty = {}
+ for diff, scores in difficulty_scores.items():
+ by_difficulty[diff] = {
+ "baseline_p5": round(mean(scores["baseline_p5"]), 4),
+ "treatment_p5": round(mean(scores["treatment_p5"]), 4),
+ }
+
+ return ABResult(
+ corpus_size=len(corpus),
+ query_count=len(queries),
+ baseline_p5=mean(baseline_p5_scores),
+ baseline_mrr=mean(baseline_mrr_scores),
+ baseline_ndcg5=mean(baseline_ndcg5_scores),
+ treatment_p5=mean(treatment_p5_scores),
+ treatment_mrr=mean(treatment_mrr_scores),
+ treatment_ndcg5=mean(treatment_ndcg5_scores),
+ by_difficulty=by_difficulty,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Report generation
+# ---------------------------------------------------------------------------
+
+
+def generate_report(result: ABResult, output_path: Path | None = None) -> dict:
+ """Generate bench_report.json content and optionally write to disk.
+
+ Args:
+ result: ABResult from run_ab_comparison.
+ output_path: If provided, write JSON report to this path.
+
+ Returns:
+ Report dict.
+ """
+
+ def relative_improvement(treatment: float, baseline: float) -> str:
+ if baseline == 0:
+ return "+inf%" if treatment > 0 else "+0.00%"
+ pct = (treatment - baseline) / baseline * 100
+ return f"{pct:+.2f}%"
+
+ report = {
+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),
+ "corpus_size": result.corpus_size,
+ "query_count": result.query_count,
+ "baseline": {
+ "p_at_5": round(result.baseline_p5, 4),
+ "mrr": round(result.baseline_mrr, 4),
+ "ndcg_at_5": round(result.baseline_ndcg5, 4),
+ },
+ "treatment": {
+ "p_at_5": round(result.treatment_p5, 4),
+ "mrr": round(result.treatment_mrr, 4),
+ "ndcg_at_5": round(result.treatment_ndcg5, 4),
+ },
+ "improvement": {
+ "p_at_5_relative": relative_improvement(
+ result.treatment_p5, result.baseline_p5
+ ),
+ "mrr_relative": relative_improvement(
+ result.treatment_mrr, result.baseline_mrr
+ ),
+ "ndcg_at_5_relative": relative_improvement(
+ result.treatment_ndcg5, result.baseline_ndcg5
+ ),
+ },
+ "by_difficulty": result.by_difficulty,
+ "gate_passed": _check_gate(result),
+ }
+
+ if output_path:
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(output_path, "w", encoding="utf-8") as f:
+ json.dump(report, f, indent=2, ensure_ascii=False)
+
+ return report
+
+
+def _check_gate(result: ABResult) -> bool:
+ """Check if treatment passes the quality gate (>=10% relative P@5 improvement)."""
+ if result.baseline_p5 == 0:
+ return result.treatment_p5 > 0
+ improvement = (result.treatment_p5 - result.baseline_p5) / result.baseline_p5
+ return improvement >= 0.10
\ No newline at end of file
diff --git a/agent/tests/memory/benchmarks/test_benchmark.py b/agent/tests/memory/benchmarks/test_benchmark.py
new file mode 100644
index 000000000..7301b2212
--- /dev/null
+++ b/agent/tests/memory/benchmarks/test_benchmark.py
@@ -0,0 +1,215 @@
+"""Pytest integration for P@5 evaluation benchmark.
+
+Tests validate that the Treatment mode (importance decay + quality scoring)
+delivers measurable improvement over the Baseline mode (pure relevance).
+
+This benchmark suite is NOT intended for CI. It requires corpus files
+(``tmp/benchmark_corpus/``) that are not version-controlled. The suite
+automatically skips when corpus files are absent.
+
+Usage:
+ python -m pytest agent/tests/memory/benchmarks/ -v
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from .runner import (
+ ABResult,
+ generate_report,
+ load_corpus,
+ load_queries,
+ retrieve_top_k,
+ run_ab_comparison,
+ tokenize,
+)
+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k
+
+# Project root for report output
+PROJECT_ROOT = Path(__file__).resolve().parents[4]
+REPORT_PATH = PROJECT_ROOT / "bench_report.json"
+
+
+# ---------------------------------------------------------------------------
+# Shared result (computed once per session)
+# ---------------------------------------------------------------------------
+
+
[email protected](scope="session")
+def ab_result(
+ memories_corpus: list[dict[str, Any]], queries_dataset: list[dict[str, Any]]
+) -> ABResult:
+ """Run A/B comparison once and share across all tests."""
+ return run_ab_comparison(memories_corpus, queries_dataset)
+
+
[email protected](scope="session")
+def report(ab_result: ABResult) -> dict:
+ """Generate and persist bench_report.json."""
+ return generate_report(ab_result, output_path=REPORT_PATH)
+
+
+# ---------------------------------------------------------------------------
+# Gate tests
+# ---------------------------------------------------------------------------
+
+
+class TestP5Gate:
+ """Quality gate: Treatment must outperform Baseline."""
+
+ def test_p5_improvement(self, ab_result: ABResult, report: dict) -> None:
+ """Verify Treatment P@5 improves over Baseline by >=10% relative."""
+ if ab_result.baseline_p5 == 0:
+ pytest.skip("Baseline P@5 is 0; relative improvement undefined")
+
+ improvement = (
+ (ab_result.treatment_p5 - ab_result.baseline_p5) / ab_result.baseline_p5
+ )
+ assert improvement >= 0.10, (
+ f"P@5 improvement {improvement:.2%} < 10% threshold. "
+ f"Baseline={ab_result.baseline_p5:.4f}, "
+ f"Treatment={ab_result.treatment_p5:.4f}"
+ )
+
+ def test_mrr_no_regression(self, ab_result: ABResult) -> None:
+ """Verify Treatment MRR does not regress vs Baseline."""
+ assert ab_result.treatment_mrr >= ab_result.baseline_mrr * 0.95, (
+ f"MRR regression: Treatment={ab_result.treatment_mrr:.4f} < "
+ f"95% of Baseline={ab_result.baseline_mrr:.4f}"
+ )
+
+ def test_ndcg5_no_regression(self, ab_result: ABResult) -> None:
+ """Verify Treatment NDCG@5 does not regress vs Baseline."""
+ assert ab_result.treatment_ndcg5 >= ab_result.baseline_ndcg5 * 0.95, (
+ f"NDCG@5 regression: Treatment={ab_result.treatment_ndcg5:.4f} < "
+ f"95% of Baseline={ab_result.baseline_ndcg5:.4f}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Difficulty-stratified tests
+# ---------------------------------------------------------------------------
+
+
+class TestDifficultyStratified:
+ """Ensure treatment benefits hard queries without hurting easy ones."""
+
+ def test_hard_queries_benefit(
+ self,
+ memories_corpus: list[dict[str, Any]],
+ queries_dataset: list[dict[str, Any]],
+ ) -> None:
+ """Verify hard queries see improved P@5 under Treatment mode."""
+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)
+ queries = load_queries(queries_dataset)
+ hard_queries = [q for q in queries if q.difficulty == "hard"]
+
+ if not hard_queries:
+ pytest.skip("No hard queries in dataset")
+
+ baseline_scores = []
+ treatment_scores = []
+
+ for q in hard_queries:
+ qt = tokenize(q.query)
+ b_results = retrieve_top_k(qt, corpus, treatment=False)
+ t_results = retrieve_top_k(
+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ baseline_scores.append(
+ precision_at_k(b_results, q.ground_truth_top5, k=5)
+ )
+ treatment_scores.append(
+ precision_at_k(t_results, q.ground_truth_top5, k=5)
+ )
+
+ baseline_mean = sum(baseline_scores) / len(baseline_scores)
+ treatment_mean = sum(treatment_scores) / len(treatment_scores)
+
+ # Hard queries should benefit from BM25 + importance weighting
+ assert treatment_mean >= baseline_mean, (
+ f"Hard queries did not benefit: "
+ f"Treatment P@5={treatment_mean:.4f} < Baseline={baseline_mean:.4f}"
+ )
+
+ def test_no_regression_easy_queries(
+ self,
+ memories_corpus: list[dict[str, Any]],
+ queries_dataset: list[dict[str, Any]],
+ ) -> None:
+ """Verify easy queries do not degrade under Treatment mode."""
+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)
+ queries = load_queries(queries_dataset)
+ easy_queries = [q for q in queries if q.difficulty == "easy"]
+
+ if not easy_queries:
+ pytest.skip("No easy queries in dataset")
+
+ baseline_scores = []
+ treatment_scores = []
+
+ for q in easy_queries:
+ qt = tokenize(q.query)
+ b_results = retrieve_top_k(qt, corpus, treatment=False)
+ t_results = retrieve_top_k(
+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ baseline_scores.append(
+ precision_at_k(b_results, q.ground_truth_top5, k=5)
+ )
+ treatment_scores.append(
+ precision_at_k(t_results, q.ground_truth_top5, k=5)
+ )
+
+ baseline_mean = sum(baseline_scores) / len(baseline_scores)
+ treatment_mean = sum(treatment_scores) / len(treatment_scores)
+
+ # Allow at most 5% relative degradation on easy queries
+ threshold = baseline_mean * 0.95
+ assert treatment_mean >= threshold, (
+ f"Easy queries regressed: "
+ f"Treatment P@5={treatment_mean:.4f} < 95% of Baseline={baseline_mean:.4f}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Report output verification
+# ---------------------------------------------------------------------------
+
+
+class TestReportOutput:
+ """Verify bench_report.json structure and content."""
+
+ def test_report_structure(self, report: dict) -> None:
+ """Verify report contains all required fields."""
+ assert "timestamp" in report
+ assert "corpus_size" in report
+ assert "query_count" in report
+ assert "baseline" in report
+ assert "treatment" in report
+ assert "improvement" in report
+ assert "by_difficulty" in report
+ assert "gate_passed" in report
+
+ def test_report_values_valid(self, report: dict) -> None:
+ """Verify report metric values are in valid ranges."""
+ for mode in ("baseline", "treatment"):
+ assert 0 <= report[mode]["p_at_5"] <= 1.0
+ assert 0 <= report[mode]["mrr"] <= 1.0
+ assert 0 <= report[mode]["ndcg_at_5"] <= 1.0
+
+ def test_report_corpus_size(self, report: dict) -> None:
+ """Verify corpus size matches expected 200."""
+ assert report["corpus_size"] == 200
+
+ def test_report_query_count(self, report: dict) -> None:
+ """Verify query count matches expected 50."""
+ assert report["query_count"] == 50
+
+ def test_report_file_written(self, report: dict) -> None:
+ """Verify bench_report.json file exists on disk."""
+ assert REPORT_PATH.exists(), f"Report not written to {REPORT_PATH}"
\ No newline at end of file