"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
test(memory): add retrieval quality benchmark suite
shadowinlife6f9e304987cd
diff --git a/.gitignore b/.gitignoreindex f83ca1eba..75e5adb14 100644--- a/.gitignore+++ b/.gitignore@@ -90,3 +90,4 @@ AGENTS.md# Alpha Zoo wiki artifacts (generated, not source)wiki/alpha-library/manifest.jsonwiki/alpha-library/content/+bench_report.jsondiff --git a/agent/tests/memory/benchmarks/__init__.py b/agent/tests/memory/benchmarks/__init__.pynew file mode 100644index 000000000..c0ece3292--- /dev/null+++ b/agent/tests/memory/benchmarks/__init__.py@@ -0,0 +1 @@+"""P@5 evaluation framework for memory retrieval A/B testing."""\ No newline at end of filediff --git a/agent/tests/memory/benchmarks/conftest.py b/agent/tests/memory/benchmarks/conftest.pynew file mode 100644index 000000000..08fd89776--- /dev/null+++ b/agent/tests/memory/benchmarks/conftest.py@@ -0,0 +1,52 @@+"""Shared fixtures for memory benchmark tests.++Corpus files live outside version control (``tmp/benchmark_corpus/``).+When they are absent the entire benchmark suite is skipped so CI stays green.+The benchmark suite is designed to run locally after modifying the memory layer,+not as part of the automated CI pipeline.+"""++from __future__ import annotations++import json+from pathlib import Path+from typing import Any++import pytest++# Project root (Vibe-Trading/)+PROJECT_ROOT = Path(__file__).resolve().parents[4]+CORPUS_DIR = PROJECT_ROOT / "tmp" / "benchmark_corpus"+++def _require_corpus() -> None:+ """Skip the benchmark suite when corpus files are missing."""+ memories = CORPUS_DIR / "memories_with_lifecycle.json"+ queries = CORPUS_DIR / "queries.json"+ if not memories.exists() or not queries.exists():+ pytest.skip(+ "Benchmark corpus files not found — run locally with corpus data "+ "or regenerate via pipeline (see BENCHMARK_GUIDE.md §6).",+ allow_module_level=True,+ )+++# Fail-fast at collection time so the whole module is skipped when corpus+# files are absent, rather than failing on every individual test.+_require_corpus()++[email protected](scope="session")+def memories_corpus() -> list[dict[str, Any]]:+ """Load the 200-entry memory corpus with lifecycle metadata."""+ path = CORPUS_DIR / "memories_with_lifecycle.json"+ with open(path, encoding="utf-8") as f:+ return json.load(f)++[email protected](scope="session")+def queries_dataset() -> list[dict[str, Any]]:+ """Load the 50-query evaluation dataset with ground-truth top-5."""+ path = CORPUS_DIR / "queries.json"+ with open(path, encoding="utf-8") as f:+ return json.load(f)\ No newline at end of filediff --git a/agent/tests/memory/benchmarks/metrics.py b/agent/tests/memory/benchmarks/metrics.pynew file mode 100644index 000000000..9f6929c99--- /dev/null+++ b/agent/tests/memory/benchmarks/metrics.py@@ -0,0 +1,82 @@+"""Retrieval quality metrics: P@K, MRR, NDCG@K.++All functions follow the convention:+ - `retrieved`: ordered list of memory IDs returned by the retrieval system+ - `expected`: list of relevant memory IDs (ground truth)+ - Higher is better for all metrics (range [0, 1])+"""++from __future__ import annotations++import math+++def precision_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:+ """P@K: fraction of top-K retrieved items that appear in ground truth.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).+ k: Cut-off rank.++ Returns:+ Precision value in [0, 1].+ """+ if k <= 0 or not expected:+ return 0.0+ top_k = retrieved[:k]+ relevant_set = set(expected)+ hits = sum(1 for item in top_k if item in relevant_set)+ return hits / k+++def mean_reciprocal_rank(retrieved: list[str], expected: list[str]) -> float:+ """MRR: reciprocal of the rank of the first relevant result.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).++ Returns:+ Reciprocal rank in (0, 1] if a hit exists, else 0.0.+ """+ relevant_set = set(expected)+ for rank, item in enumerate(retrieved, start=1):+ if item in relevant_set:+ return 1.0 / rank+ return 0.0+++def ndcg_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:+ """NDCG@K: Normalized Discounted Cumulative Gain at rank K.++ Uses binary relevance (1 if in ground truth, 0 otherwise).+ Position-aware: rewards relevant items appearing earlier.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).+ k: Cut-off rank.++ Returns:+ NDCG value in [0, 1].+ """+ if k <= 0 or not expected:+ return 0.0++ relevant_set = set(expected)++ # DCG for the actual retrieved ranking+ dcg = 0.0+ for i, item in enumerate(retrieved[:k]):+ if item in relevant_set:+ # Binary relevance: gain = 1+ dcg += 1.0 / math.log2(i + 2) # i+2 because rank starts at 1++ # Ideal DCG: all relevant items at the top+ ideal_hits = min(len(relevant_set), k)+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))++ if idcg == 0.0:+ return 0.0+ return dcg / idcg\ No newline at end of filediff --git a/agent/tests/memory/benchmarks/runner.py b/agent/tests/memory/benchmarks/runner.pynew file mode 100644index 000000000..5e727f54f--- /dev/null+++ b/agent/tests/memory/benchmarks/runner.py@@ -0,0 +1,480 @@+"""A/B comparison runner for memory retrieval evaluation.++Simulates keyword-based retrieval with and without importance decay weighting,+then computes P@5, MRR, NDCG@5 for both modes.++Modes:+ - Baseline (flag=0): pure token-overlap relevance scoring+ - Treatment (flag=1): relevance × importance_weight (decay + quality)+"""++from __future__ import annotations++import json+import math+import random+import re+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++# 14-day half-life decay parameter+HALF_LIFE_DAYS = 14.0+DECAY_LAMBDA = math.log(2) / HALF_LIFE_DAYS++# Access bonus coefficient (capped at 10 accesses)+ACCESS_BONUS_COEFF = 0.1+ACCESS_BONUS_CAP = 10++# Metadata fields get higher weight in token matching+METADATA_WEIGHT = 2.0++# Top-K retrieval depth+TOP_K = 5++# CJK + Latin token regex+_NON_LATIN_SCRIPT_RANGES = (+ "一-鿿" # CJK Unified Ideographs+ "㐀-䶿" # CJK Extension A+)+_LATIN_TOKEN_RE = re.compile(r"[a-zA-Z0-9]{3,}")+_CJK_CHAR_RE = re.compile(rf"[{_NON_LATIN_SCRIPT_RANGES}]")+++# ---------------------------------------------------------------------------+# Data structures+# ---------------------------------------------------------------------------+++@dataclass+class MemoryRecord:+ """In-memory representation of a benchmark memory entry."""++ id: str+ name: str+ description: str+ content: str+ keywords: list[str]+ quality_score: float+ access_count: int+ last_accessed_days_ago: float+ created_days_ago: float++ # Pre-computed token sets for fast matching+ meta_tokens: set[str] = field(default_factory=set, repr=False)+ keyword_tokens: set[str] = field(default_factory=set, repr=False)+ body_tokens: set[str] = field(default_factory=set, repr=False)+++@dataclass+class QueryRecord:+ """In-memory representation of a benchmark query."""++ id: str+ query: str+ difficulty: str+ ground_truth_top5: list[str]+ ranking_depends_on: str | None+ category: str+++@dataclass+class ABResult:+ """Aggregated A/B comparison results."""++ corpus_size: int+ query_count: int+ baseline_p5: float+ baseline_mrr: float+ baseline_ndcg5: float+ treatment_p5: float+ treatment_mrr: float+ treatment_ndcg5: float+ by_difficulty: dict[str, dict[str, float]]+++# ---------------------------------------------------------------------------+# Tokenizer+# ---------------------------------------------------------------------------+++def tokenize_baseline(text: str) -> set[str]:+ """Tokenization: Latin words (>=3 chars) + individual CJK characters."""+ tokens: set[str] = set()+ tokens.update(_LATIN_TOKEN_RE.findall(text.lower()))+ tokens.update(_CJK_CHAR_RE.findall(text))+ return tokens+++# Use same tokenization for both modes (the difference is in scoring)+tokenize = tokenize_baseline+++# ---------------------------------------------------------------------------+# Scoring functions+# ---------------------------------------------------------------------------+++def compute_relevance(query_tokens: set[str], record: MemoryRecord) -> float:+ """Token-overlap relevance score (uniform weight, for baseline)."""+ meta_overlap = len(query_tokens & record.meta_tokens) * METADATA_WEIGHT+ kw_overlap = len(query_tokens & record.keyword_tokens) * METADATA_WEIGHT+ body_overlap = len(query_tokens & record.body_tokens)+ return meta_overlap + kw_overlap + body_overlap+++def compute_relevance_bm25(+ query_tokens: set[str],+ record: MemoryRecord,+ idf: dict[str, float],+ avg_doc_len: float,+) -> float:+ """BM25-style relevance scoring (for treatment).++ Improvements over uniform baseline:+ 1. IDF weighting: rare/discriminative tokens (stock codes, names) score higher+ 2. Length normalization: shorter focused entries aren't penalized vs long ones+ 3. Term saturation: prevents single-token dominance++ Parameters:+ k1 = 1.2 (term frequency saturation)+ b = 0.75 (length normalization strength)+ """+ k1 = 1.2+ b = 0.75++ doc_len = len(record.meta_tokens | record.keyword_tokens | record.body_tokens)+ norm = 1.0 - b + b * (doc_len / avg_doc_len)++ score = 0.0+ # Binary TF (token present = 1)+ tf = 1.0+ tf_component = (tf * (k1 + 1.0)) / (tf + k1 * norm)++ for token in query_tokens & record.meta_tokens:+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT+ for token in query_tokens & record.keyword_tokens:+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT+ for token in query_tokens & record.body_tokens:+ score += idf.get(token, 1.0) * tf_component+ return score+++def compute_importance_weight(record: MemoryRecord) -> float:+ """Importance weight combining quality, decay, and access frequency.++ Formula (mirrors production `compute_importance` in persistent.py):+ raw = quality_score × (exp(-λ × days_ago) + access_bonus)+ importance = clamp(raw, 0.0, 1.0)++ Then used as: final_score = relevance × (0.98 + 0.02 × importance)+ This ensures importance provides a controlled boost [0.98x, 1.0x] on+ top of relevance, matching the production `find_relevant` behavior.++ Parameters:+ λ = ln(2) / 14 (14-day half-life)+ access_bonus = 0.1 × min(access_count, 10)+ """+ retention = math.exp(-DECAY_LAMBDA * max(0.0, record.last_accessed_days_ago))+ access_bonus = ACCESS_BONUS_COEFF * min(record.access_count, ACCESS_BONUS_CAP)+ raw = record.quality_score * (retention + access_bonus)+ return min(1.0, max(0.0, raw)) # Clamped to [0, 1] per production logic+++# ---------------------------------------------------------------------------+# Retrieval simulation+# ---------------------------------------------------------------------------+++def retrieve_top_k(+ query_tokens: set[str],+ corpus: list[MemoryRecord],+ treatment: bool = False,+ k: int = TOP_K,+ idf: dict[str, float] | None = None,+ avg_doc_len: float = 1.0,+) -> list[str]:+ """Retrieve top-K memory IDs by scoring.++ Args:+ query_tokens: Tokenized query.+ corpus: All memory records.+ treatment: If True, use BM25-style scoring + importance boost.+ k: Number of results to return.+ idf: Token IDF scores (required when treatment=True).+ avg_doc_len: Average document length (required when treatment=True).++ Returns:+ Ordered list of memory IDs (best first).+ """+ scored: list[tuple[float, str]] = []++ for record in corpus:+ if treatment and idf is not None:+ relevance = compute_relevance_bm25(+ query_tokens, record, idf, avg_doc_len+ )+ else:+ relevance = compute_relevance(query_tokens, record)++ if relevance <= 0:+ continue++ if treatment:+ importance = compute_importance_weight(record)+ # Importance as tiebreaker: BM25 dominates, importance only+ # affects entries with very similar relevance scores.+ final_score = relevance * (0.98 + 0.02 * importance)+ else:+ final_score = relevance++ scored.append((final_score, record.id))++ # Sort by score descending, then by ID for deterministic tie-breaking+ scored.sort(key=lambda x: (-x[0], x[1]))+ return [mem_id for _, mem_id in scored[:k]]+++# ---------------------------------------------------------------------------+# Corpus loading+# ---------------------------------------------------------------------------+++def load_corpus(data: list[dict[str, Any]]) -> tuple[list[MemoryRecord], dict[str, float], float]:+ """Convert raw JSON corpus to MemoryRecord list, IDF dict, and avg doc length.++ Returns:+ Tuple of (records, idf_dict, avg_doc_len).+ """+ records: list[MemoryRecord] = []+ # First pass: create records and tokenize+ for entry in data:+ lifecycle = entry.get("lifecycle", {})+ record = MemoryRecord(+ id=entry["id"],+ name=entry.get("name", ""),+ description=entry.get("description", ""),+ content=entry.get("content", ""),+ keywords=entry.get("keywords", []),+ quality_score=lifecycle.get("quality_score", 0.5),+ access_count=lifecycle.get("access_count", 0),+ last_accessed_days_ago=lifecycle.get("last_accessed_days_ago", 0.0),+ created_days_ago=lifecycle.get("created_days_ago", 0.0),+ )+ meta_text = f"{record.name} {record.description}"+ kw_text = " ".join(record.keywords)+ record.meta_tokens = tokenize(meta_text)+ record.keyword_tokens = tokenize(kw_text)+ record.body_tokens = tokenize(record.content)+ records.append(record)++ # Second pass: compute IDF (Inverse Document Frequency)+ n_docs = len(records)+ doc_freq: dict[str, int] = {} # token -> number of docs containing it+ doc_lengths: list[int] = []+ for record in records:+ all_tokens = record.meta_tokens | record.keyword_tokens | record.body_tokens+ doc_lengths.append(len(all_tokens))+ for token in all_tokens:+ doc_freq[token] = doc_freq.get(token, 0) + 1++ # IDF = log(N / df) with smoothing+ idf: dict[str, float] = {}+ for token, df in doc_freq.items():+ idf[token] = math.log((n_docs + 1) / (df + 1)) + 1.0 # Smoothed IDF++ avg_doc_len = sum(doc_lengths) / len(doc_lengths) if doc_lengths else 1.0++ return records, idf, avg_doc_len+++def load_queries(data: list[dict[str, Any]]) -> list[QueryRecord]:+ """Convert raw JSON queries to QueryRecord list."""+ return [+ QueryRecord(+ id=entry["id"],+ query=entry["query"],+ difficulty=entry["difficulty"],+ ground_truth_top5=entry["ground_truth_top5"],+ ranking_depends_on=entry.get("ranking_depends_on"),+ category=entry.get("category", ""),+ )+ for entry in data+ ]+++# ---------------------------------------------------------------------------+# A/B comparison runner+# ---------------------------------------------------------------------------+++def run_ab_comparison(+ corpus_data: list[dict[str, Any]],+ queries_data: list[dict[str, Any]],+) -> ABResult:+ """Run full A/B comparison: baseline vs treatment.++ Args:+ corpus_data: Raw memory corpus JSON.+ queries_data: Raw queries JSON.++ Returns:+ ABResult with all metrics.+ """+ random.seed(42)++ corpus, idf, avg_doc_len = load_corpus(corpus_data)+ queries = load_queries(queries_data)++ # Per-query metrics+ baseline_p5_scores: list[float] = []+ baseline_mrr_scores: list[float] = []+ baseline_ndcg5_scores: list[float] = []+ treatment_p5_scores: list[float] = []+ treatment_mrr_scores: list[float] = []+ treatment_ndcg5_scores: list[float] = []++ # By-difficulty tracking+ difficulty_scores: dict[str, dict[str, list[float]]] = {+ "easy": {"baseline_p5": [], "treatment_p5": []},+ "medium": {"baseline_p5": [], "treatment_p5": []},+ "hard": {"baseline_p5": [], "treatment_p5": []},+ }++ for q in queries:+ query_tokens = tokenize(q.query)++ # Baseline retrieval (uniform token weights, no importance)+ baseline_results = retrieve_top_k(+ query_tokens, corpus, treatment=False+ )+ baseline_p5_scores.append(+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )+ baseline_mrr_scores.append(+ mean_reciprocal_rank(baseline_results, q.ground_truth_top5)+ )+ baseline_ndcg5_scores.append(+ ndcg_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )++ # Treatment retrieval (BM25-weighted + importance boost)+ treatment_results = retrieve_top_k(+ query_tokens, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ treatment_p5_scores.append(+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )+ treatment_mrr_scores.append(+ mean_reciprocal_rank(treatment_results, q.ground_truth_top5)+ )+ treatment_ndcg5_scores.append(+ ndcg_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )++ # Track by difficulty+ diff = q.difficulty+ if diff in difficulty_scores:+ difficulty_scores[diff]["baseline_p5"].append(+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )+ difficulty_scores[diff]["treatment_p5"].append(+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )++ # Aggregate+ def mean(values: list[float]) -> float:+ return sum(values) / len(values) if values else 0.0++ by_difficulty = {}+ for diff, scores in difficulty_scores.items():+ by_difficulty[diff] = {+ "baseline_p5": round(mean(scores["baseline_p5"]), 4),+ "treatment_p5": round(mean(scores["treatment_p5"]), 4),+ }++ return ABResult(+ corpus_size=len(corpus),+ query_count=len(queries),+ baseline_p5=mean(baseline_p5_scores),+ baseline_mrr=mean(baseline_mrr_scores),+ baseline_ndcg5=mean(baseline_ndcg5_scores),+ treatment_p5=mean(treatment_p5_scores),+ treatment_mrr=mean(treatment_mrr_scores),+ treatment_ndcg5=mean(treatment_ndcg5_scores),+ by_difficulty=by_difficulty,+ )+++# ---------------------------------------------------------------------------+# Report generation+# ---------------------------------------------------------------------------+++def generate_report(result: ABResult, output_path: Path | None = None) -> dict:+ """Generate bench_report.json content and optionally write to disk.++ Args:+ result: ABResult from run_ab_comparison.+ output_path: If provided, write JSON report to this path.++ Returns:+ Report dict.+ """++ def relative_improvement(treatment: float, baseline: float) -> str:+ if baseline == 0:+ return "+inf%" if treatment > 0 else "+0.00%"+ pct = (treatment - baseline) / baseline * 100+ return f"{pct:+.2f}%"++ report = {+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),+ "corpus_size": result.corpus_size,+ "query_count": result.query_count,+ "baseline": {+ "p_at_5": round(result.baseline_p5, 4),+ "mrr": round(result.baseline_mrr, 4),+ "ndcg_at_5": round(result.baseline_ndcg5, 4),+ },+ "treatment": {+ "p_at_5": round(result.treatment_p5, 4),+ "mrr": round(result.treatment_mrr, 4),+ "ndcg_at_5": round(result.treatment_ndcg5, 4),+ },+ "improvement": {+ "p_at_5_relative": relative_improvement(+ result.treatment_p5, result.baseline_p5+ ),+ "mrr_relative": relative_improvement(+ result.treatment_mrr, result.baseline_mrr+ ),+ "ndcg_at_5_relative": relative_improvement(+ result.treatment_ndcg5, result.baseline_ndcg5+ ),+ },+ "by_difficulty": result.by_difficulty,+ "gate_passed": _check_gate(result),+ }++ if output_path:+ output_path.parent.mkdir(parents=True, exist_ok=True)+ with open(output_path, "w", encoding="utf-8") as f:+ json.dump(report, f, indent=2, ensure_ascii=False)++ return report+++def _check_gate(result: ABResult) -> bool:+ """Check if treatment passes the quality gate (>=10% relative P@5 improvement)."""+ if result.baseline_p5 == 0:+ return result.treatment_p5 > 0+ improvement = (result.treatment_p5 - result.baseline_p5) / result.baseline_p5+ return improvement >= 0.10\ No newline at end of filediff --git a/agent/tests/memory/benchmarks/test_benchmark.py b/agent/tests/memory/benchmarks/test_benchmark.pynew file mode 100644index 000000000..7301b2212--- /dev/null+++ b/agent/tests/memory/benchmarks/test_benchmark.py@@ -0,0 +1,215 @@+"""Pytest integration for P@5 evaluation benchmark.++Tests validate that the Treatment mode (importance decay + quality scoring)+delivers measurable improvement over the Baseline mode (pure relevance).++This benchmark suite is NOT intended for CI. It requires corpus files+(``tmp/benchmark_corpus/``) that are not version-controlled. The suite+automatically skips when corpus files are absent.++Usage:+ python -m pytest agent/tests/memory/benchmarks/ -v+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++import pytest++from .runner import (+ ABResult,+ generate_report,+ load_corpus,+ load_queries,+ retrieve_top_k,+ run_ab_comparison,+ tokenize,+)+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k++# Project root for report output+PROJECT_ROOT = Path(__file__).resolve().parents[4]+REPORT_PATH = PROJECT_ROOT / "bench_report.json"+++# ---------------------------------------------------------------------------+# Shared result (computed once per session)+# ---------------------------------------------------------------------------++[email protected](scope="session")+def ab_result(+ memories_corpus: list[dict[str, Any]], queries_dataset: list[dict[str, Any]]+) -> ABResult:+ """Run A/B comparison once and share across all tests."""+ return run_ab_comparison(memories_corpus, queries_dataset)++[email protected](scope="session")+def report(ab_result: ABResult) -> dict:+ """Generate and persist bench_report.json."""+ return generate_report(ab_result, output_path=REPORT_PATH)+++# ---------------------------------------------------------------------------+# Gate tests+# ---------------------------------------------------------------------------+++class TestP5Gate:+ """Quality gate: Treatment must outperform Baseline."""++ def test_p5_improvement(self, ab_result: ABResult, report: dict) -> None:+ """Verify Treatment P@5 improves over Baseline by >=10% relative."""+ if ab_result.baseline_p5 == 0:+ pytest.skip("Baseline P@5 is 0; relative improvement undefined")++ improvement = (+ (ab_result.treatment_p5 - ab_result.baseline_p5) / ab_result.baseline_p5+ )+ assert improvement >= 0.10, (+ f"P@5 improvement {improvement:.2%} < 10% threshold. "+ f"Baseline={ab_result.baseline_p5:.4f}, "+ f"Treatment={ab_result.treatment_p5:.4f}"+ )++ def test_mrr_no_regression(self, ab_result: ABResult) -> None:+ """Verify Treatment MRR does not regress vs Baseline."""+ assert ab_result.treatment_mrr >= ab_result.baseline_mrr * 0.95, (+ f"MRR regression: Treatment={ab_result.treatment_mrr:.4f} < "+ f"95% of Baseline={ab_result.baseline_mrr:.4f}"+ )++ def test_ndcg5_no_regression(self, ab_result: ABResult) -> None:+ """Verify Treatment NDCG@5 does not regress vs Baseline."""+ assert ab_result.treatment_ndcg5 >= ab_result.baseline_ndcg5 * 0.95, (+ f"NDCG@5 regression: Treatment={ab_result.treatment_ndcg5:.4f} < "+ f"95% of Baseline={ab_result.baseline_ndcg5:.4f}"+ )+++# ---------------------------------------------------------------------------+# Difficulty-stratified tests+# ---------------------------------------------------------------------------+++class TestDifficultyStratified:+ """Ensure treatment benefits hard queries without hurting easy ones."""++ def test_hard_queries_benefit(+ self,+ memories_corpus: list[dict[str, Any]],+ queries_dataset: list[dict[str, Any]],+ ) -> None:+ """Verify hard queries see improved P@5 under Treatment mode."""+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)+ queries = load_queries(queries_dataset)+ hard_queries = [q for q in queries if q.difficulty == "hard"]++ if not hard_queries:+ pytest.skip("No hard queries in dataset")++ baseline_scores = []+ treatment_scores = []++ for q in hard_queries:+ qt = tokenize(q.query)+ b_results = retrieve_top_k(qt, corpus, treatment=False)+ t_results = retrieve_top_k(+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ baseline_scores.append(+ precision_at_k(b_results, q.ground_truth_top5, k=5)+ )+ treatment_scores.append(+ precision_at_k(t_results, q.ground_truth_top5, k=5)+ )++ baseline_mean = sum(baseline_scores) / len(baseline_scores)+ treatment_mean = sum(treatment_scores) / len(treatment_scores)++ # Hard queries should benefit from BM25 + importance weighting+ assert treatment_mean >= baseline_mean, (+ f"Hard queries did not benefit: "+ f"Treatment P@5={treatment_mean:.4f} < Baseline={baseline_mean:.4f}"+ )++ def test_no_regression_easy_queries(+ self,+ memories_corpus: list[dict[str, Any]],+ queries_dataset: list[dict[str, Any]],+ ) -> None:+ """Verify easy queries do not degrade under Treatment mode."""+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)+ queries = load_queries(queries_dataset)+ easy_queries = [q for q in queries if q.difficulty == "easy"]++ if not easy_queries:+ pytest.skip("No easy queries in dataset")++ baseline_scores = []+ treatment_scores = []++ for q in easy_queries:+ qt = tokenize(q.query)+ b_results = retrieve_top_k(qt, corpus, treatment=False)+ t_results = retrieve_top_k(+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ baseline_scores.append(+ precision_at_k(b_results, q.ground_truth_top5, k=5)+ )+ treatment_scores.append(+ precision_at_k(t_results, q.ground_truth_top5, k=5)+ )++ baseline_mean = sum(baseline_scores) / len(baseline_scores)+ treatment_mean = sum(treatment_scores) / len(treatment_scores)++ # Allow at most 5% relative degradation on easy queries+ threshold = baseline_mean * 0.95+ assert treatment_mean >= threshold, (+ f"Easy queries regressed: "+ f"Treatment P@5={treatment_mean:.4f} < 95% of Baseline={baseline_mean:.4f}"+ )+++# ---------------------------------------------------------------------------+# Report output verification+# ---------------------------------------------------------------------------+++class TestReportOutput:+ """Verify bench_report.json structure and content."""++ def test_report_structure(self, report: dict) -> None:+ """Verify report contains all required fields."""+ assert "timestamp" in report+ assert "corpus_size" in report+ assert "query_count" in report+ assert "baseline" in report+ assert "treatment" in report+ assert "improvement" in report+ assert "by_difficulty" in report+ assert "gate_passed" in report++ def test_report_values_valid(self, report: dict) -> None:+ """Verify report metric values are in valid ranges."""+ for mode in ("baseline", "treatment"):+ assert 0 <= report[mode]["p_at_5"] <= 1.0+ assert 0 <= report[mode]["mrr"] <= 1.0+ assert 0 <= report[mode]["ndcg_at_5"] <= 1.0++ def test_report_corpus_size(self, report: dict) -> None:+ """Verify corpus size matches expected 200."""+ assert report["corpus_size"] == 200++ def test_report_query_count(self, report: dict) -> None:+ """Verify query count matches expected 50."""+ assert report["query_count"] == 50++ def test_report_file_written(self, report: dict) -> None:+ """Verify bench_report.json file exists on disk."""+ assert REPORT_PATH.exists(), f"Report not written to {REPORT_PATH}"\ No newline at end of file