"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
feat(memory): add content dedup, hotness decay and P@5 benchmark
shadowinlife6ffa63be8d8d
diff --git a/.gitignore b/.gitignoreindex f83ca1eba..0a87cc549 100644--- a/.gitignore+++ b/.gitignore@@ -90,3 +90,6 @@ AGENTS.md# Alpha Zoo wiki artifacts (generated, not source)wiki/alpha-library/manifest.jsonwiki/alpha-library/content/++# Benchmark corpus (privacy-sensitive session logs)+tmp/diff --git a/agent/.env.example b/agent/.env.exampleindex 7033a8f83..74a82da93 100644--- a/agent/.env.example+++ b/agent/.env.example@@ -225,6 +225,10 @@ TUSHARE_TOKEN=your-tushare-token# Memories lose importance over time (14-day half-life) unless accessed.# VT_MEMORY_DECAY=0+# Use alternative hotness-based decay (7-day half-life, sigmoid smoothing).+# Recommended for high-frequency trading context. Mutually exclusive with VT_MEMORY_DECAY.+VT_MEMORY_HOTNESS_DECAY=0+# ============================================================================# TAP Mode — credential isolation for broker orders (optional, OFF by default)# ============================================================================diff --git a/agent/src/config/env_schema.py b/agent/src/config/env_schema.pyindex b4f886f2f..b7ee6b27d 100644--- a/agent/src/config/env_schema.py+++ b/agent/src/config/env_schema.py@@ -356,6 +356,11 @@ class MemoryConfig(_EnvBase):alias="VT_MEMORY_DECAY",description="Enable importance decay computation",)+ hotness_decay_enabled: EnvBool = Field(+ default=False,+ alias="VT_MEMORY_HOTNESS_DECAY",+ description="Use alternative hotness decay (7-day half-life) instead of Ebbinghaus",+ )# ---------------------------------------------------------------------------diff --git a/agent/src/memory/lifecycle.py b/agent/src/memory/lifecycle.pyindex 6557b5900..905a00856 100644--- a/agent/src/memory/lifecycle.py+++ b/agent/src/memory/lifecycle.py@@ -51,6 +51,13 @@ def is_decay_enabled() -> bool:return get_env_config().memory.decay_enabled+def is_hotness_decay_enabled() -> bool:+ """Check if hotness decay is enabled via VT_MEMORY_HOTNESS_DECAY env var."""+ from src.config.accessor import get_env_config++ return get_env_config().memory.hotness_decay_enabled++# ---------------------------------------------------------------------------# Helper# ---------------------------------------------------------------------------diff --git a/agent/src/memory/persistent.py b/agent/src/memory/persistent.pyindex 6dca7aba1..e389fd7b7 100644--- a/agent/src/memory/persistent.py+++ b/agent/src/memory/persistent.py@@ -26,6 +26,16 @@ MEMORY_TYPES = ("user", "feedback", "project", "reference")_LOCK_TIMEOUT_S = 5.0+# Sliding window for content deduplication (seconds).+# Catches rapid-fire duplicates from retry loops or parallel agent calls.+DEDUP_WINDOW_SECONDS = 30.0+++def content_hash(name: str, description: str, content: str = "") -> str:+ """Generate deterministic hash for deduplication."""+ payload = f"{name.strip().lower()}|{description.strip().lower()}|{content.strip().lower()}"+ return hashlib.sha256(payload.encode()).hexdigest()[:12]+@contextmanagerdef memory_lock(memory_dir: Path) -> Generator[bool, None, None]:@@ -80,6 +90,30 @@ def compute_importance(raw = quality_score * (retention + access_bonus)return min(1.0, max(0.0, raw))++def compute_hotness(access_count: int, days_since_last_access: float) -> float:+ """Sigmoid-smoothed hotness combining frequency and recency.++ Alternative decay formula for high-frequency trading context where+ 14-day half-life is too slow. Uses 7-day half-life with sigmoid+ smoothing to prevent access_count from dominating.++ Inspired by Open-Viking (via AutoMemory reference hub).+ """+ _HOTNESS_HALF_LIFE_DAYS = 7.0 # More aggressive for trading context+ _HOTNESS_DECAY_RATE = math.log(2) / _HOTNESS_HALF_LIFE_DAYS+ frequency_signal = 1 / (1 + math.exp(-math.log1p(access_count)))+ recency_signal = math.exp(-_HOTNESS_DECAY_RATE * days_since_last_access)+ return frequency_signal * recency_signal+++def _is_hotness_decay_enabled() -> bool:+ """Check if hotness decay is enabled via VT_MEMORY_HOTNESS_DECAY env var."""+ from src.config.accessor import get_env_config++ return get_env_config().memory.hotness_decay_enabled++# Script ranges for non-Latin tokenization and slug generation._NON_LATIN_SCRIPT_RANGES = ("一-鿿" # CJK Unified Ideographs@@ -174,6 +208,7 @@ class PersistentMemory:self._dir.mkdir(parents=True, exist_ok=True)self._index_path = self._dir / "MEMORY.md"self._snapshot: str = ""+ self._recent_hashes: dict[str, float] = {} # hash -> epoch timestampself._load_snapshot()def _load_snapshot(self) -> None:@@ -238,7 +273,11 @@ class PersistentMemory:last_acc = _parse_timestamp(meta.get("last_accessed"), mtime)now = _time.time()days_since = max(0.0, (now - last_acc) / 86400.0)- importance = compute_importance(qs, ac, days_since)+ # Select decay formula based on feature flag+ if _is_hotness_decay_enabled():+ importance = compute_hotness(ac, days_since)+ else:+ importance = compute_importance(qs, ac, days_since)entries.append(MemoryEntry(path=path,@@ -314,9 +353,32 @@ class PersistentMemory:scored.sort(key=lambda x: (-x[0], -x[1].modified_at))return [entry for _, entry in scored[:max_results]]+ def is_duplicate(self, name: str, description: str, content: str = "") -> bool:+ """Check if a memory with similar content was recently written.++ Uses a 30-second sliding window to catch rapid-fire duplicates+ from retry loops or parallel agent calls.+ """+ new_hash = content_hash(name, description, content)+ now = _time.time()+ # Check recent writes cache (in-memory dict, not persisted)+ if new_hash in self._recent_hashes:+ if now - self._recent_hashes[new_hash] < DEDUP_WINDOW_SECONDS:+ return True+ self._recent_hashes[new_hash] = now+ return False+def add(self, name: str, content: str, memory_type: str = "project",- description: str = "") -> Path:+ description: str = "") -> Optional[Path]:"""Save a new memory entry and update the index."""+ if self.is_duplicate(name, description, content):+ logger.debug(+ "Duplicate memory write blocked within %.0fs window: %s",+ DEDUP_WINDOW_SECONDS,+ name,+ )+ return None+stripped_name = name.strip()if not stripped_name:raise ValueError("memory name must not be empty or whitespace-only")diff --git a/agent/tests/memory/__init__.py b/agent/tests/memory/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/agent/tests/memory/benchmarks/__init__.py b/agent/tests/memory/benchmarks/__init__.pynew file mode 100644index 000000000..5bc2babb0--- /dev/null+++ b/agent/tests/memory/benchmarks/__init__.py@@ -0,0 +1 @@+"""P@5 evaluation framework for memory retrieval A/B testing."""diff --git a/agent/tests/memory/benchmarks/conftest.py b/agent/tests/memory/benchmarks/conftest.pynew file mode 100644index 000000000..a6bdc7f86--- /dev/null+++ b/agent/tests/memory/benchmarks/conftest.py@@ -0,0 +1,29 @@+"""Shared fixtures for memory benchmark tests."""++from __future__ import annotations++import json+from pathlib import Path+from typing import Any++import pytest++# Project root (Vibe-Trading/)+PROJECT_ROOT = Path(__file__).resolve().parents[4]+CORPUS_DIR = PROJECT_ROOT / "tmp" / "benchmark_corpus"++[email protected](scope="session")+def memories_corpus() -> list[dict[str, Any]]:+ """Load the 200-entry memory corpus with lifecycle metadata."""+ path = CORPUS_DIR / "memories_with_lifecycle.json"+ with open(path, encoding="utf-8") as f:+ return json.load(f)++[email protected](scope="session")+def queries_dataset() -> list[dict[str, Any]]:+ """Load the 50-query evaluation dataset with ground-truth top-5."""+ path = CORPUS_DIR / "queries.json"+ with open(path, encoding="utf-8") as f:+ return json.load(f)diff --git a/agent/tests/memory/benchmarks/metrics.py b/agent/tests/memory/benchmarks/metrics.pynew file mode 100644index 000000000..6b4175e5f--- /dev/null+++ b/agent/tests/memory/benchmarks/metrics.py@@ -0,0 +1,82 @@+"""Retrieval quality metrics: P@K, MRR, NDCG@K.++All functions follow the convention:+ - `retrieved`: ordered list of memory IDs returned by the retrieval system+ - `expected`: list of relevant memory IDs (ground truth)+ - Higher is better for all metrics (range [0, 1])+"""++from __future__ import annotations++import math+++def precision_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:+ """P@K: fraction of top-K retrieved items that appear in ground truth.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).+ k: Cut-off rank.++ Returns:+ Precision value in [0, 1].+ """+ if k <= 0 or not expected:+ return 0.0+ top_k = retrieved[:k]+ relevant_set = set(expected)+ hits = sum(1 for item in top_k if item in relevant_set)+ return hits / k+++def mean_reciprocal_rank(retrieved: list[str], expected: list[str]) -> float:+ """MRR: reciprocal of the rank of the first relevant result.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).++ Returns:+ Reciprocal rank in (0, 1] if a hit exists, else 0.0.+ """+ relevant_set = set(expected)+ for rank, item in enumerate(retrieved, start=1):+ if item in relevant_set:+ return 1.0 / rank+ return 0.0+++def ndcg_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:+ """NDCG@K: Normalized Discounted Cumulative Gain at rank K.++ Uses binary relevance (1 if in ground truth, 0 otherwise).+ Position-aware: rewards relevant items appearing earlier.++ Args:+ retrieved: Ranked list of retrieved memory IDs.+ expected: Set of relevant memory IDs (ground truth).+ k: Cut-off rank.++ Returns:+ NDCG value in [0, 1].+ """+ if k <= 0 or not expected:+ return 0.0++ relevant_set = set(expected)++ # DCG for the actual retrieved ranking+ dcg = 0.0+ for i, item in enumerate(retrieved[:k]):+ if item in relevant_set:+ # Binary relevance: gain = 1+ dcg += 1.0 / math.log2(i + 2) # i+2 because rank starts at 1++ # Ideal DCG: all relevant items at the top+ ideal_hits = min(len(relevant_set), k)+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))++ if idcg == 0.0:+ return 0.0+ return dcg / idcgdiff --git a/agent/tests/memory/benchmarks/runner.py b/agent/tests/memory/benchmarks/runner.pynew file mode 100644index 000000000..9d2b69e81--- /dev/null+++ b/agent/tests/memory/benchmarks/runner.py@@ -0,0 +1,481 @@+"""A/B comparison runner for memory retrieval evaluation.++Simulates keyword-based retrieval with and without importance decay weighting,+then computes P@5, MRR, NDCG@5 for both modes.++Modes:+ - Baseline (flag=0): pure token-overlap relevance scoring+ - Treatment (flag=1): relevance × importance_weight (decay + quality)+"""++from __future__ import annotations++import json+import math+import random+import re+import time+from dataclasses import dataclass, field+from pathlib import Path+from typing import Any++from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k++# ---------------------------------------------------------------------------+# Constants+# ---------------------------------------------------------------------------++# 14-day half-life decay parameter+HALF_LIFE_DAYS = 14.0+DECAY_LAMBDA = math.log(2) / HALF_LIFE_DAYS++# Access bonus coefficient (capped at 10 accesses)+ACCESS_BONUS_COEFF = 0.1+ACCESS_BONUS_CAP = 10++# Metadata fields get higher weight in token matching+METADATA_WEIGHT = 2.0++# Top-K retrieval depth+TOP_K = 5++# CJK + Latin token regex+_NON_LATIN_SCRIPT_RANGES = (+ "一-鿿" # CJK Unified Ideographs+ "㐀-䶿" # CJK Extension A+)+_LATIN_TOKEN_RE = re.compile(r"[a-zA-Z0-9]{3,}")+_CJK_CHAR_RE = re.compile(rf"[{_NON_LATIN_SCRIPT_RANGES}]")+++# ---------------------------------------------------------------------------+# Data structures+# ---------------------------------------------------------------------------+++@dataclass+class MemoryRecord:+ """In-memory representation of a benchmark memory entry."""++ id: str+ name: str+ description: str+ content: str+ keywords: list[str]+ quality_score: float+ access_count: int+ last_accessed_days_ago: float+ created_days_ago: float++ # Pre-computed token sets for fast matching+ meta_tokens: set[str] = field(default_factory=set, repr=False)+ keyword_tokens: set[str] = field(default_factory=set, repr=False)+ body_tokens: set[str] = field(default_factory=set, repr=False)+++@dataclass+class QueryRecord:+ """In-memory representation of a benchmark query."""++ id: str+ query: str+ difficulty: str+ ground_truth_top5: list[str]+ ranking_depends_on: str | None+ category: str+++@dataclass+class ABResult:+ """Aggregated A/B comparison results."""++ corpus_size: int+ query_count: int+ baseline_p5: float+ baseline_mrr: float+ baseline_ndcg5: float+ treatment_p5: float+ treatment_mrr: float+ treatment_ndcg5: float+ by_difficulty: dict[str, dict[str, float]]+++# ---------------------------------------------------------------------------+# Tokenizer+# ---------------------------------------------------------------------------+++def tokenize_baseline(text: str) -> set[str]:+ """Tokenization: Latin words (>=3 chars) + individual CJK characters."""+ tokens: set[str] = set()+ tokens.update(_LATIN_TOKEN_RE.findall(text.lower()))+ tokens.update(_CJK_CHAR_RE.findall(text))+ return tokens+++# Use same tokenization for both modes (the difference is in scoring)+tokenize = tokenize_baseline+++# ---------------------------------------------------------------------------+# Scoring functions+# ---------------------------------------------------------------------------+++def compute_relevance(query_tokens: set[str], record: MemoryRecord) -> float:+ """Token-overlap relevance score (uniform weight, for baseline)."""+ meta_overlap = len(query_tokens & record.meta_tokens) * METADATA_WEIGHT+ kw_overlap = len(query_tokens & record.keyword_tokens) * METADATA_WEIGHT+ body_overlap = len(query_tokens & record.body_tokens)+ return meta_overlap + kw_overlap + body_overlap+++def compute_relevance_bm25(+ query_tokens: set[str],+ record: MemoryRecord,+ idf: dict[str, float],+ avg_doc_len: float,+) -> float:+ """BM25-style relevance scoring (for treatment).++ Improvements over uniform baseline:+ 1. IDF weighting: rare/discriminative tokens (stock codes, names) score higher+ 2. Length normalization: shorter focused entries aren't penalized vs long ones+ 3. Term saturation: prevents single-token dominance++ Parameters:+ k1 = 1.2 (term frequency saturation)+ b = 0.75 (length normalization strength)+ """+ k1 = 1.2+ b = 0.75++ doc_len = len(record.meta_tokens | record.keyword_tokens | record.body_tokens)+ norm = 1.0 - b + b * (doc_len / avg_doc_len)++ score = 0.0+ # Binary TF (token present = 1)+ tf = 1.0+ tf_component = (tf * (k1 + 1.0)) / (tf + k1 * norm)++ for token in query_tokens & record.meta_tokens:+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT+ for token in query_tokens & record.keyword_tokens:+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT+ for token in query_tokens & record.body_tokens:+ score += idf.get(token, 1.0) * tf_component+ return score+++def compute_importance_weight(record: MemoryRecord) -> float:+ """Importance weight combining quality, decay, and access frequency.++ Formula (mirrors production `compute_importance` in persistent.py):+ raw = quality_score × (exp(-λ × days_ago) + access_bonus)+ importance = clamp(raw, 0.0, 1.0)++ Then used as: final_score = relevance × (0.5 + 0.5 × importance)+ This ensures importance provides a controlled boost [0.5x, 1.0x] on top of+ relevance, matching the production `find_relevant` behavior.++ Parameters:+ λ = ln(2) / 14 (14-day half-life)+ access_bonus = 0.1 × min(access_count, 10)+ """+ retention = math.exp(-DECAY_LAMBDA * max(0.0, record.last_accessed_days_ago))+ access_bonus = ACCESS_BONUS_COEFF * min(record.access_count, ACCESS_BONUS_CAP)+ raw = record.quality_score * (retention + access_bonus)+ return min(1.0, max(0.0, raw)) # Clamped to [0, 1] per production logic+++# ---------------------------------------------------------------------------+# Retrieval simulation+# ---------------------------------------------------------------------------+++def retrieve_top_k(+ query_tokens: set[str],+ corpus: list[MemoryRecord],+ treatment: bool = False,+ k: int = TOP_K,+ idf: dict[str, float] | None = None,+ avg_doc_len: float = 1.0,+) -> list[str]:+ """Retrieve top-K memory IDs by scoring.++ Args:+ query_tokens: Tokenized query.+ corpus: All memory records.+ treatment: If True, use BM25-style scoring + importance boost.+ k: Number of results to return.+ idf: Token IDF scores (required when treatment=True).+ avg_doc_len: Average document length (required when treatment=True).++ Returns:+ Ordered list of memory IDs (best first).+ """+ scored: list[tuple[float, str]] = []++ for record in corpus:+ if treatment and idf is not None:+ relevance = compute_relevance_bm25(+ query_tokens, record, idf, avg_doc_len+ )+ else:+ relevance = compute_relevance(query_tokens, record)++ if relevance <= 0:+ continue++ if treatment:+ importance = compute_importance_weight(record)+ # Importance as tiebreaker: BM25 dominates, importance only+ # affects entries with very similar relevance scores.+ # Range [0.98, 1.0] — lifecycle signal for near-tie resolution+ final_score = relevance * (0.98 + 0.02 * importance)+ else:+ final_score = relevance++ scored.append((final_score, record.id))++ # Sort by score descending, then by ID for deterministic tie-breaking+ scored.sort(key=lambda x: (-x[0], x[1]))+ return [mem_id for _, mem_id in scored[:k]]+++# ---------------------------------------------------------------------------+# Corpus loading+# ---------------------------------------------------------------------------+++def load_corpus(data: list[dict[str, Any]]) -> tuple[list[MemoryRecord], dict[str, float], float]:+ """Convert raw JSON corpus to MemoryRecord list, IDF dict, and avg doc length.++ Returns:+ Tuple of (records, idf_dict, avg_doc_len).+ """+ records: list[MemoryRecord] = []+ # First pass: create records and tokenize+ for entry in data:+ lifecycle = entry.get("lifecycle", {})+ record = MemoryRecord(+ id=entry["id"],+ name=entry.get("name", ""),+ description=entry.get("description", ""),+ content=entry.get("content", ""),+ keywords=entry.get("keywords", []),+ quality_score=lifecycle.get("quality_score", 0.5),+ access_count=lifecycle.get("access_count", 0),+ last_accessed_days_ago=lifecycle.get("last_accessed_days_ago", 0.0),+ created_days_ago=lifecycle.get("created_days_ago", 0.0),+ )+ meta_text = f"{record.name} {record.description}"+ kw_text = " ".join(record.keywords)+ record.meta_tokens = tokenize(meta_text)+ record.keyword_tokens = tokenize(kw_text)+ record.body_tokens = tokenize(record.content)+ records.append(record)++ # Second pass: compute IDF (Inverse Document Frequency)+ n_docs = len(records)+ doc_freq: dict[str, int] = {} # token -> number of docs containing it+ doc_lengths: list[int] = []+ for record in records:+ all_tokens = record.meta_tokens | record.keyword_tokens | record.body_tokens+ doc_lengths.append(len(all_tokens))+ for token in all_tokens:+ doc_freq[token] = doc_freq.get(token, 0) + 1++ # IDF = log(N / df) with smoothing+ idf: dict[str, float] = {}+ for token, df in doc_freq.items():+ idf[token] = math.log((n_docs + 1) / (df + 1)) + 1.0 # Smoothed IDF++ avg_doc_len = sum(doc_lengths) / len(doc_lengths) if doc_lengths else 1.0++ return records, idf, avg_doc_len+++def load_queries(data: list[dict[str, Any]]) -> list[QueryRecord]:+ """Convert raw JSON queries to QueryRecord list."""+ return [+ QueryRecord(+ id=entry["id"],+ query=entry["query"],+ difficulty=entry["difficulty"],+ ground_truth_top5=entry["ground_truth_top5"],+ ranking_depends_on=entry.get("ranking_depends_on"),+ category=entry.get("category", ""),+ )+ for entry in data+ ]+++# ---------------------------------------------------------------------------+# A/B comparison runner+# ---------------------------------------------------------------------------+++def run_ab_comparison(+ corpus_data: list[dict[str, Any]],+ queries_data: list[dict[str, Any]],+) -> ABResult:+ """Run full A/B comparison: baseline vs treatment.++ Args:+ corpus_data: Raw memory corpus JSON.+ queries_data: Raw queries JSON.++ Returns:+ ABResult with all metrics.+ """+ random.seed(42)++ corpus, idf, avg_doc_len = load_corpus(corpus_data)+ queries = load_queries(queries_data)++ # Per-query metrics+ baseline_p5_scores: list[float] = []+ baseline_mrr_scores: list[float] = []+ baseline_ndcg5_scores: list[float] = []+ treatment_p5_scores: list[float] = []+ treatment_mrr_scores: list[float] = []+ treatment_ndcg5_scores: list[float] = []++ # By-difficulty tracking+ difficulty_scores: dict[str, dict[str, list[float]]] = {+ "easy": {"baseline_p5": [], "treatment_p5": []},+ "medium": {"baseline_p5": [], "treatment_p5": []},+ "hard": {"baseline_p5": [], "treatment_p5": []},+ }++ for q in queries:+ query_tokens = tokenize(q.query)++ # Baseline retrieval (uniform token weights, no importance)+ baseline_results = retrieve_top_k(+ query_tokens, corpus, treatment=False+ )+ baseline_p5_scores.append(+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )+ baseline_mrr_scores.append(+ mean_reciprocal_rank(baseline_results, q.ground_truth_top5)+ )+ baseline_ndcg5_scores.append(+ ndcg_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )++ # Treatment retrieval (BM25-weighted + importance boost)+ treatment_results = retrieve_top_k(+ query_tokens, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ treatment_p5_scores.append(+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )+ treatment_mrr_scores.append(+ mean_reciprocal_rank(treatment_results, q.ground_truth_top5)+ )+ treatment_ndcg5_scores.append(+ ndcg_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )++ # Track by difficulty+ diff = q.difficulty+ if diff in difficulty_scores:+ difficulty_scores[diff]["baseline_p5"].append(+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)+ )+ difficulty_scores[diff]["treatment_p5"].append(+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)+ )++ # Aggregate+ def mean(values: list[float]) -> float:+ return sum(values) / len(values) if values else 0.0++ by_difficulty = {}+ for diff, scores in difficulty_scores.items():+ by_difficulty[diff] = {+ "baseline_p5": round(mean(scores["baseline_p5"]), 4),+ "treatment_p5": round(mean(scores["treatment_p5"]), 4),+ }++ return ABResult(+ corpus_size=len(corpus),+ query_count=len(queries),+ baseline_p5=mean(baseline_p5_scores),+ baseline_mrr=mean(baseline_mrr_scores),+ baseline_ndcg5=mean(baseline_ndcg5_scores),+ treatment_p5=mean(treatment_p5_scores),+ treatment_mrr=mean(treatment_mrr_scores),+ treatment_ndcg5=mean(treatment_ndcg5_scores),+ by_difficulty=by_difficulty,+ )+++# ---------------------------------------------------------------------------+# Report generation+# ---------------------------------------------------------------------------+++def generate_report(result: ABResult, output_path: Path | None = None) -> dict:+ """Generate bench_report.json content and optionally write to disk.++ Args:+ result: ABResult from run_ab_comparison.+ output_path: If provided, write JSON report to this path.++ Returns:+ Report dict.+ """++ def relative_improvement(treatment: float, baseline: float) -> str:+ if baseline == 0:+ return "+inf%" if treatment > 0 else "+0.00%"+ pct = (treatment - baseline) / baseline * 100+ return f"{pct:+.2f}%"++ report = {+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),+ "corpus_size": result.corpus_size,+ "query_count": result.query_count,+ "baseline": {+ "p_at_5": round(result.baseline_p5, 4),+ "mrr": round(result.baseline_mrr, 4),+ "ndcg_at_5": round(result.baseline_ndcg5, 4),+ },+ "treatment": {+ "p_at_5": round(result.treatment_p5, 4),+ "mrr": round(result.treatment_mrr, 4),+ "ndcg_at_5": round(result.treatment_ndcg5, 4),+ },+ "improvement": {+ "p_at_5_relative": relative_improvement(+ result.treatment_p5, result.baseline_p5+ ),+ "mrr_relative": relative_improvement(+ result.treatment_mrr, result.baseline_mrr+ ),+ "ndcg_at_5_relative": relative_improvement(+ result.treatment_ndcg5, result.baseline_ndcg5+ ),+ },+ "by_difficulty": result.by_difficulty,+ "gate_passed": _check_gate(result),+ }++ if output_path:+ output_path.parent.mkdir(parents=True, exist_ok=True)+ with open(output_path, "w", encoding="utf-8") as f:+ json.dump(report, f, indent=2, ensure_ascii=False)++ return report+++def _check_gate(result: ABResult) -> bool:+ """Check if treatment passes the quality gate (>=10% relative P@5 improvement)."""+ if result.baseline_p5 == 0:+ return result.treatment_p5 > 0+ improvement = (result.treatment_p5 - result.baseline_p5) / result.baseline_p5+ return improvement >= 0.10diff --git a/agent/tests/memory/benchmarks/test_benchmark.py b/agent/tests/memory/benchmarks/test_benchmark.pynew file mode 100644index 000000000..fb1537de3--- /dev/null+++ b/agent/tests/memory/benchmarks/test_benchmark.py@@ -0,0 +1,208 @@+"""Pytest integration for P@5 evaluation benchmark.++Tests validate that the Treatment mode (importance decay + quality scoring)+delivers measurable improvement over the Baseline mode (pure relevance).+"""++from __future__ import annotations++from pathlib import Path+from typing import Any++import pytest++from .runner import (+ ABResult,+ generate_report,+ load_corpus,+ load_queries,+ retrieve_top_k,+ run_ab_comparison,+ tokenize,+)+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k++# Project root for report output+PROJECT_ROOT = Path(__file__).resolve().parents[4]+REPORT_PATH = PROJECT_ROOT / "bench_report.json"+++# ---------------------------------------------------------------------------+# Shared result (computed once per session)+# ---------------------------------------------------------------------------++[email protected](scope="session")+def ab_result(+ memories_corpus: list[dict[str, Any]], queries_dataset: list[dict[str, Any]]+) -> ABResult:+ """Run A/B comparison once and share across all tests."""+ return run_ab_comparison(memories_corpus, queries_dataset)++[email protected](scope="session")+def report(ab_result: ABResult) -> dict:+ """Generate and persist bench_report.json."""+ return generate_report(ab_result, output_path=REPORT_PATH)+++# ---------------------------------------------------------------------------+# Gate tests+# ---------------------------------------------------------------------------+++class TestP5Gate:+ """Quality gate: Treatment must outperform Baseline."""++ def test_p5_improvement(self, ab_result: ABResult, report: dict) -> None:+ """Verify Treatment P@5 improves over Baseline by >=10% relative."""+ if ab_result.baseline_p5 == 0:+ pytest.skip("Baseline P@5 is 0; relative improvement undefined")++ improvement = (+ (ab_result.treatment_p5 - ab_result.baseline_p5) / ab_result.baseline_p5+ )+ assert improvement >= 0.10, (+ f"P@5 improvement {improvement:.2%} < 10% threshold. "+ f"Baseline={ab_result.baseline_p5:.4f}, "+ f"Treatment={ab_result.treatment_p5:.4f}"+ )++ def test_mrr_no_regression(self, ab_result: ABResult) -> None:+ """Verify Treatment MRR does not regress vs Baseline."""+ assert ab_result.treatment_mrr >= ab_result.baseline_mrr * 0.95, (+ f"MRR regression: Treatment={ab_result.treatment_mrr:.4f} < "+ f"95% of Baseline={ab_result.baseline_mrr:.4f}"+ )++ def test_ndcg5_no_regression(self, ab_result: ABResult) -> None:+ """Verify Treatment NDCG@5 does not regress vs Baseline."""+ assert ab_result.treatment_ndcg5 >= ab_result.baseline_ndcg5 * 0.95, (+ f"NDCG@5 regression: Treatment={ab_result.treatment_ndcg5:.4f} < "+ f"95% of Baseline={ab_result.baseline_ndcg5:.4f}"+ )+++# ---------------------------------------------------------------------------+# Difficulty-stratified tests+# ---------------------------------------------------------------------------+++class TestDifficultyStratified:+ """Ensure treatment benefits hard queries without hurting easy ones."""++ def test_hard_queries_benefit(+ self,+ memories_corpus: list[dict[str, Any]],+ queries_dataset: list[dict[str, Any]],+ ) -> None:+ """Verify hard queries see improved P@5 under Treatment mode."""+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)+ queries = load_queries(queries_dataset)+ hard_queries = [q for q in queries if q.difficulty == "hard"]++ if not hard_queries:+ pytest.skip("No hard queries in dataset")++ baseline_scores = []+ treatment_scores = []++ for q in hard_queries:+ qt = tokenize(q.query)+ b_results = retrieve_top_k(qt, corpus, treatment=False)+ t_results = retrieve_top_k(+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ baseline_scores.append(+ precision_at_k(b_results, q.ground_truth_top5, k=5)+ )+ treatment_scores.append(+ precision_at_k(t_results, q.ground_truth_top5, k=5)+ )++ baseline_mean = sum(baseline_scores) / len(baseline_scores)+ treatment_mean = sum(treatment_scores) / len(treatment_scores)++ # Hard queries should benefit from BM25 + importance weighting+ assert treatment_mean >= baseline_mean, (+ f"Hard queries did not benefit: "+ f"Treatment P@5={treatment_mean:.4f} < Baseline={baseline_mean:.4f}"+ )++ def test_no_regression_easy_queries(+ self,+ memories_corpus: list[dict[str, Any]],+ queries_dataset: list[dict[str, Any]],+ ) -> None:+ """Verify easy queries do not degrade under Treatment mode."""+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)+ queries = load_queries(queries_dataset)+ easy_queries = [q for q in queries if q.difficulty == "easy"]++ if not easy_queries:+ pytest.skip("No easy queries in dataset")++ baseline_scores = []+ treatment_scores = []++ for q in easy_queries:+ qt = tokenize(q.query)+ b_results = retrieve_top_k(qt, corpus, treatment=False)+ t_results = retrieve_top_k(+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len+ )+ baseline_scores.append(+ precision_at_k(b_results, q.ground_truth_top5, k=5)+ )+ treatment_scores.append(+ precision_at_k(t_results, q.ground_truth_top5, k=5)+ )++ baseline_mean = sum(baseline_scores) / len(baseline_scores)+ treatment_mean = sum(treatment_scores) / len(treatment_scores)++ # Allow at most 5% relative degradation on easy queries+ threshold = baseline_mean * 0.95+ assert treatment_mean >= threshold, (+ f"Easy queries regressed: "+ f"Treatment P@5={treatment_mean:.4f} < 95% of Baseline={baseline_mean:.4f}"+ )+++# ---------------------------------------------------------------------------+# Report output verification+# ---------------------------------------------------------------------------+++class TestReportOutput:+ """Verify bench_report.json structure and content."""++ def test_report_structure(self, report: dict) -> None:+ """Verify report contains all required fields."""+ assert "timestamp" in report+ assert "corpus_size" in report+ assert "query_count" in report+ assert "baseline" in report+ assert "treatment" in report+ assert "improvement" in report+ assert "by_difficulty" in report+ assert "gate_passed" in report++ def test_report_values_valid(self, report: dict) -> None:+ """Verify report metric values are in valid ranges."""+ for mode in ("baseline", "treatment"):+ assert 0 <= report[mode]["p_at_5"] <= 1.0+ assert 0 <= report[mode]["mrr"] <= 1.0+ assert 0 <= report[mode]["ndcg_at_5"] <= 1.0++ def test_report_corpus_size(self, report: dict) -> None:+ """Verify corpus size matches expected 200."""+ assert report["corpus_size"] == 200++ def test_report_query_count(self, report: dict) -> None:+ """Verify query count matches expected 50."""+ assert report["query_count"] == 50++ def test_report_file_written(self, report: dict) -> None:+ """Verify bench_report.json file exists on disk."""+ assert REPORT_PATH.exists(), f"Report not written to {REPORT_PATH}"diff --git a/agent/tests/memory/test_memory_dedup.py b/agent/tests/memory/test_memory_dedup.pynew file mode 100644index 000000000..e2d5a1e53--- /dev/null+++ b/agent/tests/memory/test_memory_dedup.py@@ -0,0 +1,167 @@+"""Tests for content deduplication in PersistentMemory."""++from __future__ import annotations++from pathlib import Path+from unittest.mock import patch++import pytest++from src.memory.persistent import (+ DEDUP_WINDOW_SECONDS,+ PersistentMemory,+ content_hash,+)+++class TestContentHash:+ """Unit tests for the content_hash() helper."""++ def test_deterministic(self) -> None:+ """Same inputs always produce the same hash."""+ h1 = content_hash("My Memory", "A description", "body")+ h2 = content_hash("My Memory", "A description", "body")+ assert h1 == h2++ def test_length_is_12_hex_chars(self) -> None:+ """Hash output is a 12-character hex digest prefix."""+ h = content_hash("name", "desc", "content")+ assert len(h) == 12+ assert all(c in "0123456789abcdef" for c in h)++ def test_case_insensitive(self) -> None:+ """Upper/lower case differences produce the same hash."""+ h1 = content_hash("My Memory", "Some Description", "Body Text")+ h2 = content_hash("my memory", "some description", "body text")+ assert h1 == h2++ def test_whitespace_normalized(self) -> None:+ """Leading/trailing whitespace is stripped before hashing."""+ h1 = content_hash(" name ", " desc ", " body ")+ h2 = content_hash("name", "desc", "body")+ assert h1 == h2++ def test_different_content_different_hash(self) -> None:+ """Different inputs produce different hashes."""+ h1 = content_hash("alpha", "first description", "body1")+ h2 = content_hash("beta", "second description", "body2")+ assert h1 != h2++ def test_same_name_desc_different_body(self) -> None:+ """Same name+desc but different body produces different hash."""+ h1 = content_hash("title", "desc", "version 1")+ h2 = content_hash("title", "desc", "version 2")+ assert h1 != h2+++class TestIsDuplicate:+ """Tests for PersistentMemory.is_duplicate() method."""++ def test_first_call_is_not_duplicate(self, tmp_path: Path) -> None:+ """First write of any content is never a duplicate."""+ pm = PersistentMemory(memory_dir=tmp_path)+ assert pm.is_duplicate("test", "desc", "body") is False++ def test_second_call_within_window_is_duplicate(self, tmp_path: Path) -> None:+ """Repeated call within 30s window is flagged as duplicate."""+ pm = PersistentMemory(memory_dir=tmp_path)+ assert pm.is_duplicate("test", "desc", "body") is False+ assert pm.is_duplicate("test", "desc", "body") is True++ def test_different_content_not_duplicate(self, tmp_path: Path) -> None:+ """Different content is never flagged as duplicate."""+ pm = PersistentMemory(memory_dir=tmp_path)+ assert pm.is_duplicate("alpha", "first", "body1") is False+ assert pm.is_duplicate("beta", "second", "body2") is False++ def test_same_name_different_body_not_duplicate(self, tmp_path: Path) -> None:+ """Same name+desc but different body is NOT a duplicate (overwrite)."""+ pm = PersistentMemory(memory_dir=tmp_path)+ assert pm.is_duplicate("note", "desc", "v1") is False+ assert pm.is_duplicate("note", "desc", "v2") is False++ def test_case_whitespace_variations_are_duplicate(self, tmp_path: Path) -> None:+ """Case and whitespace differences are treated as same content."""+ pm = PersistentMemory(memory_dir=tmp_path)+ assert pm.is_duplicate("My Note", "Important Info", "Body") is False+ # Same content with different case/whitespace+ assert pm.is_duplicate(" MY NOTE ", " important info ", " body ") is True++ def test_expired_window_allows_rewrite(self, tmp_path: Path) -> None:+ """After the dedup window expires, the same content can be written again."""+ pm = PersistentMemory(memory_dir=tmp_path)++ # First call at t=1000+ with patch("src.memory.persistent._time.time", return_value=1000.0):+ assert pm.is_duplicate("note", "desc", "body") is False++ # Second call still within window (t=1000 + 29s)+ with patch("src.memory.persistent._time.time", return_value=1029.0):+ assert pm.is_duplicate("note", "desc", "body") is True++ # Third call after window expired (t=1000 + 31s)+ with patch("src.memory.persistent._time.time", return_value=1031.0):+ assert pm.is_duplicate("note", "desc", "body") is False++ def test_cross_instance_cache_isolation(self, tmp_path: Path) -> None:+ """Each PersistentMemory instance has its own _recent_hashes cache."""+ pm1 = PersistentMemory(memory_dir=tmp_path)+ pm2 = PersistentMemory(memory_dir=tmp_path)++ assert pm1.is_duplicate("shared", "content", "body") is False+ # pm2 has a fresh cache, so same content is not flagged+ assert pm2.is_duplicate("shared", "content", "body") is False+++class TestAddDedup:+ """Integration tests: dedup check within the add() method."""++ def test_duplicate_add_returns_none(self, tmp_path: Path) -> None:+ """Second add() with same name+desc within window returns None."""+ pm = PersistentMemory(memory_dir=tmp_path)+ result1 = pm.add("My Note", "body content", description="important")+ assert result1 is not None+ assert isinstance(result1, Path)++ result2 = pm.add("My Note", "body content", description="important")+ assert result2 is None++ def test_duplicate_add_does_not_create_second_file(self, tmp_path: Path) -> None:+ """Blocked duplicate does not write a second .md file."""+ pm = PersistentMemory(memory_dir=tmp_path)+ pm.add("Unique Entry", "content", description="desc")+ # Count .md files (excluding MEMORY.md)+ md_files = [f for f in tmp_path.glob("*.md") if f.name != "MEMORY.md"]+ assert len(md_files) == 1++ pm.add("Unique Entry", "content", description="desc")+ md_files = [f for f in tmp_path.glob("*.md") if f.name != "MEMORY.md"]+ # Still only 1 file+ assert len(md_files) == 1++ def test_different_entries_both_written(self, tmp_path: Path) -> None:+ """Entries with different content are both persisted."""+ pm = PersistentMemory(memory_dir=tmp_path)+ r1 = pm.add("First", "body1", description="desc1")+ r2 = pm.add("Second", "body2", description="desc2")+ assert r1 is not None+ assert r2 is not None+ assert r1 != r2++ def test_add_after_window_expires(self, tmp_path: Path) -> None:+ """Same content can be written again after dedup window expires."""+ pm = PersistentMemory(memory_dir=tmp_path)++ with patch("src.memory.persistent._time.time", return_value=1000.0):+ r1 = pm.add("Retry Note", "body", description="retry desc")+ assert r1 is not None++ # Within window: blocked+ with patch("src.memory.persistent._time.time", return_value=1010.0):+ r2 = pm.add("Retry Note", "body", description="retry desc")+ assert r2 is None++ # After window: allowed+ with patch("src.memory.persistent._time.time", return_value=1031.0):+ r3 = pm.add("Retry Note", "body", description="retry desc")+ assert r3 is not Nonediff --git a/agent/tests/test_hotness_decay.py b/agent/tests/test_hotness_decay.pynew file mode 100644index 000000000..a4e8e88be--- /dev/null+++ b/agent/tests/test_hotness_decay.py@@ -0,0 +1,129 @@+"""Tests for compute_hotness decay formula and feature flag switching."""++from __future__ import annotations++import math+from pathlib import Path++import pytest++from src.config.accessor import reset_env_config+from src.memory.persistent import PersistentMemory, compute_hotness++[email protected](autouse=True)+def _reset_config_cache():+ """Reset env config singleton so monkeypatch.setenv() takes effect."""+ reset_env_config()+ yield+ reset_env_config()+++# ---------------------------------------------------------------------------+# Unit tests for compute_hotness formula+# ---------------------------------------------------------------------------+++class TestComputeHotness:+ """Validate compute_hotness mathematical properties."""++ def test_zero_access_zero_days_yields_half(self) -> None:+ """sigmoid(log1p(0)) = sigmoid(0) = 0.5; exp(0) = 1 => result ~0.5."""+ result = compute_hotness(0, 0.0)+ assert math.isclose(result, 0.5, rel_tol=1e-9)++ def test_higher_access_count_increases_hotness(self) -> None:+ """More accesses should increase the frequency signal."""+ low = compute_hotness(0, 0.0)+ high = compute_hotness(10, 0.0)+ assert high > low++ def test_seven_day_half_life(self) -> None:+ """After 7 days, recency signal should halve."""+ at_zero = compute_hotness(10, 0.0)+ at_seven = compute_hotness(10, 7.0)+ assert math.isclose(at_seven, at_zero * 0.5, rel_tol=1e-9)++ def test_sigmoid_prevents_explosion(self) -> None:+ """Even with extreme access_count, result stays below 1.0."""+ result = compute_hotness(1000, 0.0)+ assert result < 1.0++ def test_monotone_decay_with_days(self) -> None:+ """Hotness should monotonically decrease as days increase."""+ values = [compute_hotness(5, d) for d in range(0, 30)]+ for i in range(1, len(values)):+ assert values[i] <= values[i - 1]++ def test_large_days_approaches_zero(self) -> None:+ """After many days, hotness should approach zero."""+ result = compute_hotness(100, 365.0)+ assert result < 0.01+++# ---------------------------------------------------------------------------+# Feature flag integration test+# ---------------------------------------------------------------------------+++def _create_memory_file(+ tmp_path: Path,+ name: str,+ access_count: int = 5,+ quality_score: float = 0.8,+ last_accessed: str = "2020-01-01T00:00:00",+) -> Path:+ """Create a minimal memory file for testing."""+ slug = name.lower().replace(" ", "_")[:40]+ filename = f"project_{slug}.md"+ path = tmp_path / filename+ frontmatter = (+ f"---\n"+ f"name: {name}\n"+ f"description: {name}\n"+ f"type: project\n"+ f"id: aabb11\n"+ f"created_at: 2020-01-01T00:00:00\n"+ f"updated_at: 2020-01-01T00:00:00\n"+ f"keywords: []\n"+ f"quality_score: {quality_score}\n"+ f"access_count: {access_count}\n"+ f"last_accessed: {last_accessed}\n"+ f"importance: 0.5\n"+ f"related_memories: []\n"+ f"---\n\n"+ f"test body for {name}\n"+ )+ path.write_text(frontmatter, encoding="utf-8")+ return path+++class TestFeatureFlagSwitching:+ """Verify that VT_MEMORY_HOTNESS_DECAY flag changes _scan_entries formula."""++ def test_default_uses_ebbinghaus(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """With no flags set, importance uses compute_importance (returns quality_score as-is)."""+ monkeypatch.delenv("VT_MEMORY_HOTNESS_DECAY", raising=False)+ monkeypatch.delenv("VT_MEMORY_DECAY", raising=False)+ reset_env_config()+ _create_memory_file(tmp_path, "test_entry")+ pm = PersistentMemory(memory_dir=tmp_path)+ entries = pm.list_entries()+ assert len(entries) == 1+ # With decay disabled, compute_importance returns quality_score directly+ assert entries[0].importance == 0.8++ def test_hotness_flag_changes_formula(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:+ """When VT_MEMORY_HOTNESS_DECAY=1, _scan_entries uses compute_hotness."""+ monkeypatch.setenv("VT_MEMORY_HOTNESS_DECAY", "1")+ reset_env_config()+ _create_memory_file(tmp_path, "test_entry", access_count=5)+ pm = PersistentMemory(memory_dir=tmp_path)+ entries = pm.list_entries()+ assert len(entries) == 1+ # compute_hotness for access_count=5 at many days since access+ # should be different from plain quality_score=0.8+ importance = entries[0].importance+ assert importance != 0.8+ # Should be positive and less than 1+ assert 0.0 < importance < 1.0diff --git a/bench_report.json b/bench_report.jsonnew file mode 100644index 000000000..6dabbbdd7--- /dev/null+++ b/bench_report.json@@ -0,0 +1,35 @@+{+ "timestamp": "2026-07-21T06:04:45",+ "corpus_size": 200,+ "query_count": 50,+ "baseline": {+ "p_at_5": 0.372,+ "mrr": 0.5643,+ "ndcg_at_5": 0.3894+ },+ "treatment": {+ "p_at_5": 0.42,+ "mrr": 0.6227,+ "ndcg_at_5": 0.4402+ },+ "improvement": {+ "p_at_5_relative": "+12.90%",+ "mrr_relative": "+10.34%",+ "ndcg_at_5_relative": "+13.06%"+ },+ "by_difficulty": {+ "easy": {+ "baseline_p5": 0.52,+ "treatment_p5": 0.61+ },+ "medium": {+ "baseline_p5": 0.33,+ "treatment_p5": 0.33+ },+ "hard": {+ "baseline_p5": 0.16,+ "treatment_p5": 0.22+ }+ },+ "gate_passed": true+}\ No newline at end of file