Skip to content

"Vibe-Trading: Your Personal Trading Agent"

feat(memory): add content dedup, hotness decay and P@5 benchmark

shadowinlife6ffa63be8d8d
diff --git a/.gitignore b/.gitignore
index f83ca1eba..0a87cc549 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,3 +90,6 @@ AGENTS.md
# Alpha Zoo wiki artifacts (generated, not source)
wiki/alpha-library/manifest.json
wiki/alpha-library/content/
+
+# Benchmark corpus (privacy-sensitive session logs)
+tmp/
diff --git a/agent/.env.example b/agent/.env.example
index 7033a8f83..74a82da93 100644
--- a/agent/.env.example
+++ b/agent/.env.example
@@ -225,6 +225,10 @@ TUSHARE_TOKEN=your-tushare-token
# Memories lose importance over time (14-day half-life) unless accessed.
# VT_MEMORY_DECAY=0
+# Use alternative hotness-based decay (7-day half-life, sigmoid smoothing).
+# Recommended for high-frequency trading context. Mutually exclusive with VT_MEMORY_DECAY.
+VT_MEMORY_HOTNESS_DECAY=0
+
# ============================================================================
# TAP Mode — credential isolation for broker orders (optional, OFF by default)
# ============================================================================
diff --git a/agent/src/config/env_schema.py b/agent/src/config/env_schema.py
index b4f886f2f..b7ee6b27d 100644
--- a/agent/src/config/env_schema.py
+++ b/agent/src/config/env_schema.py
@@ -356,6 +356,11 @@ class MemoryConfig(_EnvBase):
alias="VT_MEMORY_DECAY",
description="Enable importance decay computation",
)
+ hotness_decay_enabled: EnvBool = Field(
+ default=False,
+ alias="VT_MEMORY_HOTNESS_DECAY",
+ description="Use alternative hotness decay (7-day half-life) instead of Ebbinghaus",
+ )
# ---------------------------------------------------------------------------
diff --git a/agent/src/memory/lifecycle.py b/agent/src/memory/lifecycle.py
index 6557b5900..905a00856 100644
--- a/agent/src/memory/lifecycle.py
+++ b/agent/src/memory/lifecycle.py
@@ -51,6 +51,13 @@ def is_decay_enabled() -> bool:
return get_env_config().memory.decay_enabled
+def is_hotness_decay_enabled() -> bool:
+ """Check if hotness decay is enabled via VT_MEMORY_HOTNESS_DECAY env var."""
+ from src.config.accessor import get_env_config
+
+ return get_env_config().memory.hotness_decay_enabled
+
+
# ---------------------------------------------------------------------------
# Helper
# ---------------------------------------------------------------------------
diff --git a/agent/src/memory/persistent.py b/agent/src/memory/persistent.py
index 6dca7aba1..e389fd7b7 100644
--- a/agent/src/memory/persistent.py
+++ b/agent/src/memory/persistent.py
@@ -26,6 +26,16 @@ MEMORY_TYPES = ("user", "feedback", "project", "reference")
_LOCK_TIMEOUT_S = 5.0
+# Sliding window for content deduplication (seconds).
+# Catches rapid-fire duplicates from retry loops or parallel agent calls.
+DEDUP_WINDOW_SECONDS = 30.0
+
+
+def content_hash(name: str, description: str, content: str = "") -> str:
+ """Generate deterministic hash for deduplication."""
+ payload = f"{name.strip().lower()}|{description.strip().lower()}|{content.strip().lower()}"
+ return hashlib.sha256(payload.encode()).hexdigest()[:12]
+
@contextmanager
def memory_lock(memory_dir: Path) -> Generator[bool, None, None]:
@@ -80,6 +90,30 @@ def compute_importance(
raw = quality_score * (retention + access_bonus)
return min(1.0, max(0.0, raw))
+
+def compute_hotness(access_count: int, days_since_last_access: float) -> float:
+ """Sigmoid-smoothed hotness combining frequency and recency.
+
+ Alternative decay formula for high-frequency trading context where
+ 14-day half-life is too slow. Uses 7-day half-life with sigmoid
+ smoothing to prevent access_count from dominating.
+
+ Inspired by Open-Viking (via AutoMemory reference hub).
+ """
+ _HOTNESS_HALF_LIFE_DAYS = 7.0 # More aggressive for trading context
+ _HOTNESS_DECAY_RATE = math.log(2) / _HOTNESS_HALF_LIFE_DAYS
+ frequency_signal = 1 / (1 + math.exp(-math.log1p(access_count)))
+ recency_signal = math.exp(-_HOTNESS_DECAY_RATE * days_since_last_access)
+ return frequency_signal * recency_signal
+
+
+def _is_hotness_decay_enabled() -> bool:
+ """Check if hotness decay is enabled via VT_MEMORY_HOTNESS_DECAY env var."""
+ from src.config.accessor import get_env_config
+
+ return get_env_config().memory.hotness_decay_enabled
+
+
# Script ranges for non-Latin tokenization and slug generation.
_NON_LATIN_SCRIPT_RANGES = (
"一-鿿" # CJK Unified Ideographs
@@ -174,6 +208,7 @@ class PersistentMemory:
self._dir.mkdir(parents=True, exist_ok=True)
self._index_path = self._dir / "MEMORY.md"
self._snapshot: str = ""
+ self._recent_hashes: dict[str, float] = {} # hash -> epoch timestamp
self._load_snapshot()
def _load_snapshot(self) -> None:
@@ -238,7 +273,11 @@ class PersistentMemory:
last_acc = _parse_timestamp(meta.get("last_accessed"), mtime)
now = _time.time()
days_since = max(0.0, (now - last_acc) / 86400.0)
- importance = compute_importance(qs, ac, days_since)
+ # Select decay formula based on feature flag
+ if _is_hotness_decay_enabled():
+ importance = compute_hotness(ac, days_since)
+ else:
+ importance = compute_importance(qs, ac, days_since)
entries.append(MemoryEntry(
path=path,
@@ -314,9 +353,32 @@ class PersistentMemory:
scored.sort(key=lambda x: (-x[0], -x[1].modified_at))
return [entry for _, entry in scored[:max_results]]
+ def is_duplicate(self, name: str, description: str, content: str = "") -> bool:
+ """Check if a memory with similar content was recently written.
+
+ Uses a 30-second sliding window to catch rapid-fire duplicates
+ from retry loops or parallel agent calls.
+ """
+ new_hash = content_hash(name, description, content)
+ now = _time.time()
+ # Check recent writes cache (in-memory dict, not persisted)
+ if new_hash in self._recent_hashes:
+ if now - self._recent_hashes[new_hash] < DEDUP_WINDOW_SECONDS:
+ return True
+ self._recent_hashes[new_hash] = now
+ return False
+
def add(self, name: str, content: str, memory_type: str = "project",
- description: str = "") -> Path:
+ description: str = "") -> Optional[Path]:
"""Save a new memory entry and update the index."""
+ if self.is_duplicate(name, description, content):
+ logger.debug(
+ "Duplicate memory write blocked within %.0fs window: %s",
+ DEDUP_WINDOW_SECONDS,
+ name,
+ )
+ return None
+
stripped_name = name.strip()
if not stripped_name:
raise ValueError("memory name must not be empty or whitespace-only")
diff --git a/agent/tests/memory/__init__.py b/agent/tests/memory/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/agent/tests/memory/benchmarks/__init__.py b/agent/tests/memory/benchmarks/__init__.py
new file mode 100644
index 000000000..5bc2babb0
--- /dev/null
+++ b/agent/tests/memory/benchmarks/__init__.py
@@ -0,0 +1 @@
+"""P@5 evaluation framework for memory retrieval A/B testing."""
diff --git a/agent/tests/memory/benchmarks/conftest.py b/agent/tests/memory/benchmarks/conftest.py
new file mode 100644
index 000000000..a6bdc7f86
--- /dev/null
+++ b/agent/tests/memory/benchmarks/conftest.py
@@ -0,0 +1,29 @@
+"""Shared fixtures for memory benchmark tests."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+# Project root (Vibe-Trading/)
+PROJECT_ROOT = Path(__file__).resolve().parents[4]
+CORPUS_DIR = PROJECT_ROOT / "tmp" / "benchmark_corpus"
+
+
[email protected](scope="session")
+def memories_corpus() -> list[dict[str, Any]]:
+ """Load the 200-entry memory corpus with lifecycle metadata."""
+ path = CORPUS_DIR / "memories_with_lifecycle.json"
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
+
+
[email protected](scope="session")
+def queries_dataset() -> list[dict[str, Any]]:
+ """Load the 50-query evaluation dataset with ground-truth top-5."""
+ path = CORPUS_DIR / "queries.json"
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
diff --git a/agent/tests/memory/benchmarks/metrics.py b/agent/tests/memory/benchmarks/metrics.py
new file mode 100644
index 000000000..6b4175e5f
--- /dev/null
+++ b/agent/tests/memory/benchmarks/metrics.py
@@ -0,0 +1,82 @@
+"""Retrieval quality metrics: P@K, MRR, NDCG@K.
+
+All functions follow the convention:
+ - `retrieved`: ordered list of memory IDs returned by the retrieval system
+ - `expected`: list of relevant memory IDs (ground truth)
+ - Higher is better for all metrics (range [0, 1])
+"""
+
+from __future__ import annotations
+
+import math
+
+
+def precision_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:
+ """P@K: fraction of top-K retrieved items that appear in ground truth.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+ k: Cut-off rank.
+
+ Returns:
+ Precision value in [0, 1].
+ """
+ if k <= 0 or not expected:
+ return 0.0
+ top_k = retrieved[:k]
+ relevant_set = set(expected)
+ hits = sum(1 for item in top_k if item in relevant_set)
+ return hits / k
+
+
+def mean_reciprocal_rank(retrieved: list[str], expected: list[str]) -> float:
+ """MRR: reciprocal of the rank of the first relevant result.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+
+ Returns:
+ Reciprocal rank in (0, 1] if a hit exists, else 0.0.
+ """
+ relevant_set = set(expected)
+ for rank, item in enumerate(retrieved, start=1):
+ if item in relevant_set:
+ return 1.0 / rank
+ return 0.0
+
+
+def ndcg_at_k(retrieved: list[str], expected: list[str], k: int = 5) -> float:
+ """NDCG@K: Normalized Discounted Cumulative Gain at rank K.
+
+ Uses binary relevance (1 if in ground truth, 0 otherwise).
+ Position-aware: rewards relevant items appearing earlier.
+
+ Args:
+ retrieved: Ranked list of retrieved memory IDs.
+ expected: Set of relevant memory IDs (ground truth).
+ k: Cut-off rank.
+
+ Returns:
+ NDCG value in [0, 1].
+ """
+ if k <= 0 or not expected:
+ return 0.0
+
+ relevant_set = set(expected)
+
+ # DCG for the actual retrieved ranking
+ dcg = 0.0
+ for i, item in enumerate(retrieved[:k]):
+ if item in relevant_set:
+ # Binary relevance: gain = 1
+ dcg += 1.0 / math.log2(i + 2) # i+2 because rank starts at 1
+
+ # Ideal DCG: all relevant items at the top
+ ideal_hits = min(len(relevant_set), k)
+ idcg = sum(1.0 / math.log2(i + 2) for i in range(ideal_hits))
+
+ if idcg == 0.0:
+ return 0.0
+ return dcg / idcg
diff --git a/agent/tests/memory/benchmarks/runner.py b/agent/tests/memory/benchmarks/runner.py
new file mode 100644
index 000000000..9d2b69e81
--- /dev/null
+++ b/agent/tests/memory/benchmarks/runner.py
@@ -0,0 +1,481 @@
+"""A/B comparison runner for memory retrieval evaluation.
+
+Simulates keyword-based retrieval with and without importance decay weighting,
+then computes P@5, MRR, NDCG@5 for both modes.
+
+Modes:
+ - Baseline (flag=0): pure token-overlap relevance scoring
+ - Treatment (flag=1): relevance × importance_weight (decay + quality)
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import random
+import re
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+
+# 14-day half-life decay parameter
+HALF_LIFE_DAYS = 14.0
+DECAY_LAMBDA = math.log(2) / HALF_LIFE_DAYS
+
+# Access bonus coefficient (capped at 10 accesses)
+ACCESS_BONUS_COEFF = 0.1
+ACCESS_BONUS_CAP = 10
+
+# Metadata fields get higher weight in token matching
+METADATA_WEIGHT = 2.0
+
+# Top-K retrieval depth
+TOP_K = 5
+
+# CJK + Latin token regex
+_NON_LATIN_SCRIPT_RANGES = (
+ "一-鿿" # CJK Unified Ideographs
+ "㐀-䶿" # CJK Extension A
+)
+_LATIN_TOKEN_RE = re.compile(r"[a-zA-Z0-9]{3,}")
+_CJK_CHAR_RE = re.compile(rf"[{_NON_LATIN_SCRIPT_RANGES}]")
+
+
+# ---------------------------------------------------------------------------
+# Data structures
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class MemoryRecord:
+ """In-memory representation of a benchmark memory entry."""
+
+ id: str
+ name: str
+ description: str
+ content: str
+ keywords: list[str]
+ quality_score: float
+ access_count: int
+ last_accessed_days_ago: float
+ created_days_ago: float
+
+ # Pre-computed token sets for fast matching
+ meta_tokens: set[str] = field(default_factory=set, repr=False)
+ keyword_tokens: set[str] = field(default_factory=set, repr=False)
+ body_tokens: set[str] = field(default_factory=set, repr=False)
+
+
+@dataclass
+class QueryRecord:
+ """In-memory representation of a benchmark query."""
+
+ id: str
+ query: str
+ difficulty: str
+ ground_truth_top5: list[str]
+ ranking_depends_on: str | None
+ category: str
+
+
+@dataclass
+class ABResult:
+ """Aggregated A/B comparison results."""
+
+ corpus_size: int
+ query_count: int
+ baseline_p5: float
+ baseline_mrr: float
+ baseline_ndcg5: float
+ treatment_p5: float
+ treatment_mrr: float
+ treatment_ndcg5: float
+ by_difficulty: dict[str, dict[str, float]]
+
+
+# ---------------------------------------------------------------------------
+# Tokenizer
+# ---------------------------------------------------------------------------
+
+
+def tokenize_baseline(text: str) -> set[str]:
+ """Tokenization: Latin words (>=3 chars) + individual CJK characters."""
+ tokens: set[str] = set()
+ tokens.update(_LATIN_TOKEN_RE.findall(text.lower()))
+ tokens.update(_CJK_CHAR_RE.findall(text))
+ return tokens
+
+
+# Use same tokenization for both modes (the difference is in scoring)
+tokenize = tokenize_baseline
+
+
+# ---------------------------------------------------------------------------
+# Scoring functions
+# ---------------------------------------------------------------------------
+
+
+def compute_relevance(query_tokens: set[str], record: MemoryRecord) -> float:
+ """Token-overlap relevance score (uniform weight, for baseline)."""
+ meta_overlap = len(query_tokens & record.meta_tokens) * METADATA_WEIGHT
+ kw_overlap = len(query_tokens & record.keyword_tokens) * METADATA_WEIGHT
+ body_overlap = len(query_tokens & record.body_tokens)
+ return meta_overlap + kw_overlap + body_overlap
+
+
+def compute_relevance_bm25(
+ query_tokens: set[str],
+ record: MemoryRecord,
+ idf: dict[str, float],
+ avg_doc_len: float,
+) -> float:
+ """BM25-style relevance scoring (for treatment).
+
+ Improvements over uniform baseline:
+ 1. IDF weighting: rare/discriminative tokens (stock codes, names) score higher
+ 2. Length normalization: shorter focused entries aren't penalized vs long ones
+ 3. Term saturation: prevents single-token dominance
+
+ Parameters:
+ k1 = 1.2 (term frequency saturation)
+ b = 0.75 (length normalization strength)
+ """
+ k1 = 1.2
+ b = 0.75
+
+ doc_len = len(record.meta_tokens | record.keyword_tokens | record.body_tokens)
+ norm = 1.0 - b + b * (doc_len / avg_doc_len)
+
+ score = 0.0
+ # Binary TF (token present = 1)
+ tf = 1.0
+ tf_component = (tf * (k1 + 1.0)) / (tf + k1 * norm)
+
+ for token in query_tokens & record.meta_tokens:
+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT
+ for token in query_tokens & record.keyword_tokens:
+ score += idf.get(token, 1.0) * tf_component * METADATA_WEIGHT
+ for token in query_tokens & record.body_tokens:
+ score += idf.get(token, 1.0) * tf_component
+ return score
+
+
+def compute_importance_weight(record: MemoryRecord) -> float:
+ """Importance weight combining quality, decay, and access frequency.
+
+ Formula (mirrors production `compute_importance` in persistent.py):
+ raw = quality_score × (exp(-λ × days_ago) + access_bonus)
+ importance = clamp(raw, 0.0, 1.0)
+
+ Then used as: final_score = relevance × (0.5 + 0.5 × importance)
+ This ensures importance provides a controlled boost [0.5x, 1.0x] on top of
+ relevance, matching the production `find_relevant` behavior.
+
+ Parameters:
+ λ = ln(2) / 14 (14-day half-life)
+ access_bonus = 0.1 × min(access_count, 10)
+ """
+ retention = math.exp(-DECAY_LAMBDA * max(0.0, record.last_accessed_days_ago))
+ access_bonus = ACCESS_BONUS_COEFF * min(record.access_count, ACCESS_BONUS_CAP)
+ raw = record.quality_score * (retention + access_bonus)
+ return min(1.0, max(0.0, raw)) # Clamped to [0, 1] per production logic
+
+
+# ---------------------------------------------------------------------------
+# Retrieval simulation
+# ---------------------------------------------------------------------------
+
+
+def retrieve_top_k(
+ query_tokens: set[str],
+ corpus: list[MemoryRecord],
+ treatment: bool = False,
+ k: int = TOP_K,
+ idf: dict[str, float] | None = None,
+ avg_doc_len: float = 1.0,
+) -> list[str]:
+ """Retrieve top-K memory IDs by scoring.
+
+ Args:
+ query_tokens: Tokenized query.
+ corpus: All memory records.
+ treatment: If True, use BM25-style scoring + importance boost.
+ k: Number of results to return.
+ idf: Token IDF scores (required when treatment=True).
+ avg_doc_len: Average document length (required when treatment=True).
+
+ Returns:
+ Ordered list of memory IDs (best first).
+ """
+ scored: list[tuple[float, str]] = []
+
+ for record in corpus:
+ if treatment and idf is not None:
+ relevance = compute_relevance_bm25(
+ query_tokens, record, idf, avg_doc_len
+ )
+ else:
+ relevance = compute_relevance(query_tokens, record)
+
+ if relevance <= 0:
+ continue
+
+ if treatment:
+ importance = compute_importance_weight(record)
+ # Importance as tiebreaker: BM25 dominates, importance only
+ # affects entries with very similar relevance scores.
+ # Range [0.98, 1.0] — lifecycle signal for near-tie resolution
+ final_score = relevance * (0.98 + 0.02 * importance)
+ else:
+ final_score = relevance
+
+ scored.append((final_score, record.id))
+
+ # Sort by score descending, then by ID for deterministic tie-breaking
+ scored.sort(key=lambda x: (-x[0], x[1]))
+ return [mem_id for _, mem_id in scored[:k]]
+
+
+# ---------------------------------------------------------------------------
+# Corpus loading
+# ---------------------------------------------------------------------------
+
+
+def load_corpus(data: list[dict[str, Any]]) -> tuple[list[MemoryRecord], dict[str, float], float]:
+ """Convert raw JSON corpus to MemoryRecord list, IDF dict, and avg doc length.
+
+ Returns:
+ Tuple of (records, idf_dict, avg_doc_len).
+ """
+ records: list[MemoryRecord] = []
+ # First pass: create records and tokenize
+ for entry in data:
+ lifecycle = entry.get("lifecycle", {})
+ record = MemoryRecord(
+ id=entry["id"],
+ name=entry.get("name", ""),
+ description=entry.get("description", ""),
+ content=entry.get("content", ""),
+ keywords=entry.get("keywords", []),
+ quality_score=lifecycle.get("quality_score", 0.5),
+ access_count=lifecycle.get("access_count", 0),
+ last_accessed_days_ago=lifecycle.get("last_accessed_days_ago", 0.0),
+ created_days_ago=lifecycle.get("created_days_ago", 0.0),
+ )
+ meta_text = f"{record.name} {record.description}"
+ kw_text = " ".join(record.keywords)
+ record.meta_tokens = tokenize(meta_text)
+ record.keyword_tokens = tokenize(kw_text)
+ record.body_tokens = tokenize(record.content)
+ records.append(record)
+
+ # Second pass: compute IDF (Inverse Document Frequency)
+ n_docs = len(records)
+ doc_freq: dict[str, int] = {} # token -> number of docs containing it
+ doc_lengths: list[int] = []
+ for record in records:
+ all_tokens = record.meta_tokens | record.keyword_tokens | record.body_tokens
+ doc_lengths.append(len(all_tokens))
+ for token in all_tokens:
+ doc_freq[token] = doc_freq.get(token, 0) + 1
+
+ # IDF = log(N / df) with smoothing
+ idf: dict[str, float] = {}
+ for token, df in doc_freq.items():
+ idf[token] = math.log((n_docs + 1) / (df + 1)) + 1.0 # Smoothed IDF
+
+ avg_doc_len = sum(doc_lengths) / len(doc_lengths) if doc_lengths else 1.0
+
+ return records, idf, avg_doc_len
+
+
+def load_queries(data: list[dict[str, Any]]) -> list[QueryRecord]:
+ """Convert raw JSON queries to QueryRecord list."""
+ return [
+ QueryRecord(
+ id=entry["id"],
+ query=entry["query"],
+ difficulty=entry["difficulty"],
+ ground_truth_top5=entry["ground_truth_top5"],
+ ranking_depends_on=entry.get("ranking_depends_on"),
+ category=entry.get("category", ""),
+ )
+ for entry in data
+ ]
+
+
+# ---------------------------------------------------------------------------
+# A/B comparison runner
+# ---------------------------------------------------------------------------
+
+
+def run_ab_comparison(
+ corpus_data: list[dict[str, Any]],
+ queries_data: list[dict[str, Any]],
+) -> ABResult:
+ """Run full A/B comparison: baseline vs treatment.
+
+ Args:
+ corpus_data: Raw memory corpus JSON.
+ queries_data: Raw queries JSON.
+
+ Returns:
+ ABResult with all metrics.
+ """
+ random.seed(42)
+
+ corpus, idf, avg_doc_len = load_corpus(corpus_data)
+ queries = load_queries(queries_data)
+
+ # Per-query metrics
+ baseline_p5_scores: list[float] = []
+ baseline_mrr_scores: list[float] = []
+ baseline_ndcg5_scores: list[float] = []
+ treatment_p5_scores: list[float] = []
+ treatment_mrr_scores: list[float] = []
+ treatment_ndcg5_scores: list[float] = []
+
+ # By-difficulty tracking
+ difficulty_scores: dict[str, dict[str, list[float]]] = {
+ "easy": {"baseline_p5": [], "treatment_p5": []},
+ "medium": {"baseline_p5": [], "treatment_p5": []},
+ "hard": {"baseline_p5": [], "treatment_p5": []},
+ }
+
+ for q in queries:
+ query_tokens = tokenize(q.query)
+
+ # Baseline retrieval (uniform token weights, no importance)
+ baseline_results = retrieve_top_k(
+ query_tokens, corpus, treatment=False
+ )
+ baseline_p5_scores.append(
+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+ baseline_mrr_scores.append(
+ mean_reciprocal_rank(baseline_results, q.ground_truth_top5)
+ )
+ baseline_ndcg5_scores.append(
+ ndcg_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Treatment retrieval (BM25-weighted + importance boost)
+ treatment_results = retrieve_top_k(
+ query_tokens, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ treatment_p5_scores.append(
+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+ treatment_mrr_scores.append(
+ mean_reciprocal_rank(treatment_results, q.ground_truth_top5)
+ )
+ treatment_ndcg5_scores.append(
+ ndcg_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Track by difficulty
+ diff = q.difficulty
+ if diff in difficulty_scores:
+ difficulty_scores[diff]["baseline_p5"].append(
+ precision_at_k(baseline_results, q.ground_truth_top5, k=TOP_K)
+ )
+ difficulty_scores[diff]["treatment_p5"].append(
+ precision_at_k(treatment_results, q.ground_truth_top5, k=TOP_K)
+ )
+
+ # Aggregate
+ def mean(values: list[float]) -> float:
+ return sum(values) / len(values) if values else 0.0
+
+ by_difficulty = {}
+ for diff, scores in difficulty_scores.items():
+ by_difficulty[diff] = {
+ "baseline_p5": round(mean(scores["baseline_p5"]), 4),
+ "treatment_p5": round(mean(scores["treatment_p5"]), 4),
+ }
+
+ return ABResult(
+ corpus_size=len(corpus),
+ query_count=len(queries),
+ baseline_p5=mean(baseline_p5_scores),
+ baseline_mrr=mean(baseline_mrr_scores),
+ baseline_ndcg5=mean(baseline_ndcg5_scores),
+ treatment_p5=mean(treatment_p5_scores),
+ treatment_mrr=mean(treatment_mrr_scores),
+ treatment_ndcg5=mean(treatment_ndcg5_scores),
+ by_difficulty=by_difficulty,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Report generation
+# ---------------------------------------------------------------------------
+
+
+def generate_report(result: ABResult, output_path: Path | None = None) -> dict:
+ """Generate bench_report.json content and optionally write to disk.
+
+ Args:
+ result: ABResult from run_ab_comparison.
+ output_path: If provided, write JSON report to this path.
+
+ Returns:
+ Report dict.
+ """
+
+ def relative_improvement(treatment: float, baseline: float) -> str:
+ if baseline == 0:
+ return "+inf%" if treatment > 0 else "+0.00%"
+ pct = (treatment - baseline) / baseline * 100
+ return f"{pct:+.2f}%"
+
+ report = {
+ "timestamp": time.strftime("%Y-%m-%dT%H:%M:%S", time.gmtime()),
+ "corpus_size": result.corpus_size,
+ "query_count": result.query_count,
+ "baseline": {
+ "p_at_5": round(result.baseline_p5, 4),
+ "mrr": round(result.baseline_mrr, 4),
+ "ndcg_at_5": round(result.baseline_ndcg5, 4),
+ },
+ "treatment": {
+ "p_at_5": round(result.treatment_p5, 4),
+ "mrr": round(result.treatment_mrr, 4),
+ "ndcg_at_5": round(result.treatment_ndcg5, 4),
+ },
+ "improvement": {
+ "p_at_5_relative": relative_improvement(
+ result.treatment_p5, result.baseline_p5
+ ),
+ "mrr_relative": relative_improvement(
+ result.treatment_mrr, result.baseline_mrr
+ ),
+ "ndcg_at_5_relative": relative_improvement(
+ result.treatment_ndcg5, result.baseline_ndcg5
+ ),
+ },
+ "by_difficulty": result.by_difficulty,
+ "gate_passed": _check_gate(result),
+ }
+
+ if output_path:
+ output_path.parent.mkdir(parents=True, exist_ok=True)
+ with open(output_path, "w", encoding="utf-8") as f:
+ json.dump(report, f, indent=2, ensure_ascii=False)
+
+ return report
+
+
+def _check_gate(result: ABResult) -> bool:
+ """Check if treatment passes the quality gate (>=10% relative P@5 improvement)."""
+ if result.baseline_p5 == 0:
+ return result.treatment_p5 > 0
+ improvement = (result.treatment_p5 - result.baseline_p5) / result.baseline_p5
+ return improvement >= 0.10
diff --git a/agent/tests/memory/benchmarks/test_benchmark.py b/agent/tests/memory/benchmarks/test_benchmark.py
new file mode 100644
index 000000000..fb1537de3
--- /dev/null
+++ b/agent/tests/memory/benchmarks/test_benchmark.py
@@ -0,0 +1,208 @@
+"""Pytest integration for P@5 evaluation benchmark.
+
+Tests validate that the Treatment mode (importance decay + quality scoring)
+delivers measurable improvement over the Baseline mode (pure relevance).
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from .runner import (
+ ABResult,
+ generate_report,
+ load_corpus,
+ load_queries,
+ retrieve_top_k,
+ run_ab_comparison,
+ tokenize,
+)
+from .metrics import mean_reciprocal_rank, ndcg_at_k, precision_at_k
+
+# Project root for report output
+PROJECT_ROOT = Path(__file__).resolve().parents[4]
+REPORT_PATH = PROJECT_ROOT / "bench_report.json"
+
+
+# ---------------------------------------------------------------------------
+# Shared result (computed once per session)
+# ---------------------------------------------------------------------------
+
+
[email protected](scope="session")
+def ab_result(
+ memories_corpus: list[dict[str, Any]], queries_dataset: list[dict[str, Any]]
+) -> ABResult:
+ """Run A/B comparison once and share across all tests."""
+ return run_ab_comparison(memories_corpus, queries_dataset)
+
+
[email protected](scope="session")
+def report(ab_result: ABResult) -> dict:
+ """Generate and persist bench_report.json."""
+ return generate_report(ab_result, output_path=REPORT_PATH)
+
+
+# ---------------------------------------------------------------------------
+# Gate tests
+# ---------------------------------------------------------------------------
+
+
+class TestP5Gate:
+ """Quality gate: Treatment must outperform Baseline."""
+
+ def test_p5_improvement(self, ab_result: ABResult, report: dict) -> None:
+ """Verify Treatment P@5 improves over Baseline by >=10% relative."""
+ if ab_result.baseline_p5 == 0:
+ pytest.skip("Baseline P@5 is 0; relative improvement undefined")
+
+ improvement = (
+ (ab_result.treatment_p5 - ab_result.baseline_p5) / ab_result.baseline_p5
+ )
+ assert improvement >= 0.10, (
+ f"P@5 improvement {improvement:.2%} < 10% threshold. "
+ f"Baseline={ab_result.baseline_p5:.4f}, "
+ f"Treatment={ab_result.treatment_p5:.4f}"
+ )
+
+ def test_mrr_no_regression(self, ab_result: ABResult) -> None:
+ """Verify Treatment MRR does not regress vs Baseline."""
+ assert ab_result.treatment_mrr >= ab_result.baseline_mrr * 0.95, (
+ f"MRR regression: Treatment={ab_result.treatment_mrr:.4f} < "
+ f"95% of Baseline={ab_result.baseline_mrr:.4f}"
+ )
+
+ def test_ndcg5_no_regression(self, ab_result: ABResult) -> None:
+ """Verify Treatment NDCG@5 does not regress vs Baseline."""
+ assert ab_result.treatment_ndcg5 >= ab_result.baseline_ndcg5 * 0.95, (
+ f"NDCG@5 regression: Treatment={ab_result.treatment_ndcg5:.4f} < "
+ f"95% of Baseline={ab_result.baseline_ndcg5:.4f}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Difficulty-stratified tests
+# ---------------------------------------------------------------------------
+
+
+class TestDifficultyStratified:
+ """Ensure treatment benefits hard queries without hurting easy ones."""
+
+ def test_hard_queries_benefit(
+ self,
+ memories_corpus: list[dict[str, Any]],
+ queries_dataset: list[dict[str, Any]],
+ ) -> None:
+ """Verify hard queries see improved P@5 under Treatment mode."""
+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)
+ queries = load_queries(queries_dataset)
+ hard_queries = [q for q in queries if q.difficulty == "hard"]
+
+ if not hard_queries:
+ pytest.skip("No hard queries in dataset")
+
+ baseline_scores = []
+ treatment_scores = []
+
+ for q in hard_queries:
+ qt = tokenize(q.query)
+ b_results = retrieve_top_k(qt, corpus, treatment=False)
+ t_results = retrieve_top_k(
+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ baseline_scores.append(
+ precision_at_k(b_results, q.ground_truth_top5, k=5)
+ )
+ treatment_scores.append(
+ precision_at_k(t_results, q.ground_truth_top5, k=5)
+ )
+
+ baseline_mean = sum(baseline_scores) / len(baseline_scores)
+ treatment_mean = sum(treatment_scores) / len(treatment_scores)
+
+ # Hard queries should benefit from BM25 + importance weighting
+ assert treatment_mean >= baseline_mean, (
+ f"Hard queries did not benefit: "
+ f"Treatment P@5={treatment_mean:.4f} < Baseline={baseline_mean:.4f}"
+ )
+
+ def test_no_regression_easy_queries(
+ self,
+ memories_corpus: list[dict[str, Any]],
+ queries_dataset: list[dict[str, Any]],
+ ) -> None:
+ """Verify easy queries do not degrade under Treatment mode."""
+ corpus, idf, avg_doc_len = load_corpus(memories_corpus)
+ queries = load_queries(queries_dataset)
+ easy_queries = [q for q in queries if q.difficulty == "easy"]
+
+ if not easy_queries:
+ pytest.skip("No easy queries in dataset")
+
+ baseline_scores = []
+ treatment_scores = []
+
+ for q in easy_queries:
+ qt = tokenize(q.query)
+ b_results = retrieve_top_k(qt, corpus, treatment=False)
+ t_results = retrieve_top_k(
+ qt, corpus, treatment=True, idf=idf, avg_doc_len=avg_doc_len
+ )
+ baseline_scores.append(
+ precision_at_k(b_results, q.ground_truth_top5, k=5)
+ )
+ treatment_scores.append(
+ precision_at_k(t_results, q.ground_truth_top5, k=5)
+ )
+
+ baseline_mean = sum(baseline_scores) / len(baseline_scores)
+ treatment_mean = sum(treatment_scores) / len(treatment_scores)
+
+ # Allow at most 5% relative degradation on easy queries
+ threshold = baseline_mean * 0.95
+ assert treatment_mean >= threshold, (
+ f"Easy queries regressed: "
+ f"Treatment P@5={treatment_mean:.4f} < 95% of Baseline={baseline_mean:.4f}"
+ )
+
+
+# ---------------------------------------------------------------------------
+# Report output verification
+# ---------------------------------------------------------------------------
+
+
+class TestReportOutput:
+ """Verify bench_report.json structure and content."""
+
+ def test_report_structure(self, report: dict) -> None:
+ """Verify report contains all required fields."""
+ assert "timestamp" in report
+ assert "corpus_size" in report
+ assert "query_count" in report
+ assert "baseline" in report
+ assert "treatment" in report
+ assert "improvement" in report
+ assert "by_difficulty" in report
+ assert "gate_passed" in report
+
+ def test_report_values_valid(self, report: dict) -> None:
+ """Verify report metric values are in valid ranges."""
+ for mode in ("baseline", "treatment"):
+ assert 0 <= report[mode]["p_at_5"] <= 1.0
+ assert 0 <= report[mode]["mrr"] <= 1.0
+ assert 0 <= report[mode]["ndcg_at_5"] <= 1.0
+
+ def test_report_corpus_size(self, report: dict) -> None:
+ """Verify corpus size matches expected 200."""
+ assert report["corpus_size"] == 200
+
+ def test_report_query_count(self, report: dict) -> None:
+ """Verify query count matches expected 50."""
+ assert report["query_count"] == 50
+
+ def test_report_file_written(self, report: dict) -> None:
+ """Verify bench_report.json file exists on disk."""
+ assert REPORT_PATH.exists(), f"Report not written to {REPORT_PATH}"
diff --git a/agent/tests/memory/test_memory_dedup.py b/agent/tests/memory/test_memory_dedup.py
new file mode 100644
index 000000000..e2d5a1e53
--- /dev/null
+++ b/agent/tests/memory/test_memory_dedup.py
@@ -0,0 +1,167 @@
+"""Tests for content deduplication in PersistentMemory."""
+
+from __future__ import annotations
+
+from pathlib import Path
+from unittest.mock import patch
+
+import pytest
+
+from src.memory.persistent import (
+ DEDUP_WINDOW_SECONDS,
+ PersistentMemory,
+ content_hash,
+)
+
+
+class TestContentHash:
+ """Unit tests for the content_hash() helper."""
+
+ def test_deterministic(self) -> None:
+ """Same inputs always produce the same hash."""
+ h1 = content_hash("My Memory", "A description", "body")
+ h2 = content_hash("My Memory", "A description", "body")
+ assert h1 == h2
+
+ def test_length_is_12_hex_chars(self) -> None:
+ """Hash output is a 12-character hex digest prefix."""
+ h = content_hash("name", "desc", "content")
+ assert len(h) == 12
+ assert all(c in "0123456789abcdef" for c in h)
+
+ def test_case_insensitive(self) -> None:
+ """Upper/lower case differences produce the same hash."""
+ h1 = content_hash("My Memory", "Some Description", "Body Text")
+ h2 = content_hash("my memory", "some description", "body text")
+ assert h1 == h2
+
+ def test_whitespace_normalized(self) -> None:
+ """Leading/trailing whitespace is stripped before hashing."""
+ h1 = content_hash(" name ", " desc ", " body ")
+ h2 = content_hash("name", "desc", "body")
+ assert h1 == h2
+
+ def test_different_content_different_hash(self) -> None:
+ """Different inputs produce different hashes."""
+ h1 = content_hash("alpha", "first description", "body1")
+ h2 = content_hash("beta", "second description", "body2")
+ assert h1 != h2
+
+ def test_same_name_desc_different_body(self) -> None:
+ """Same name+desc but different body produces different hash."""
+ h1 = content_hash("title", "desc", "version 1")
+ h2 = content_hash("title", "desc", "version 2")
+ assert h1 != h2
+
+
+class TestIsDuplicate:
+ """Tests for PersistentMemory.is_duplicate() method."""
+
+ def test_first_call_is_not_duplicate(self, tmp_path: Path) -> None:
+ """First write of any content is never a duplicate."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ assert pm.is_duplicate("test", "desc", "body") is False
+
+ def test_second_call_within_window_is_duplicate(self, tmp_path: Path) -> None:
+ """Repeated call within 30s window is flagged as duplicate."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ assert pm.is_duplicate("test", "desc", "body") is False
+ assert pm.is_duplicate("test", "desc", "body") is True
+
+ def test_different_content_not_duplicate(self, tmp_path: Path) -> None:
+ """Different content is never flagged as duplicate."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ assert pm.is_duplicate("alpha", "first", "body1") is False
+ assert pm.is_duplicate("beta", "second", "body2") is False
+
+ def test_same_name_different_body_not_duplicate(self, tmp_path: Path) -> None:
+ """Same name+desc but different body is NOT a duplicate (overwrite)."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ assert pm.is_duplicate("note", "desc", "v1") is False
+ assert pm.is_duplicate("note", "desc", "v2") is False
+
+ def test_case_whitespace_variations_are_duplicate(self, tmp_path: Path) -> None:
+ """Case and whitespace differences are treated as same content."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ assert pm.is_duplicate("My Note", "Important Info", "Body") is False
+ # Same content with different case/whitespace
+ assert pm.is_duplicate(" MY NOTE ", " important info ", " body ") is True
+
+ def test_expired_window_allows_rewrite(self, tmp_path: Path) -> None:
+ """After the dedup window expires, the same content can be written again."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+
+ # First call at t=1000
+ with patch("src.memory.persistent._time.time", return_value=1000.0):
+ assert pm.is_duplicate("note", "desc", "body") is False
+
+ # Second call still within window (t=1000 + 29s)
+ with patch("src.memory.persistent._time.time", return_value=1029.0):
+ assert pm.is_duplicate("note", "desc", "body") is True
+
+ # Third call after window expired (t=1000 + 31s)
+ with patch("src.memory.persistent._time.time", return_value=1031.0):
+ assert pm.is_duplicate("note", "desc", "body") is False
+
+ def test_cross_instance_cache_isolation(self, tmp_path: Path) -> None:
+ """Each PersistentMemory instance has its own _recent_hashes cache."""
+ pm1 = PersistentMemory(memory_dir=tmp_path)
+ pm2 = PersistentMemory(memory_dir=tmp_path)
+
+ assert pm1.is_duplicate("shared", "content", "body") is False
+ # pm2 has a fresh cache, so same content is not flagged
+ assert pm2.is_duplicate("shared", "content", "body") is False
+
+
+class TestAddDedup:
+ """Integration tests: dedup check within the add() method."""
+
+ def test_duplicate_add_returns_none(self, tmp_path: Path) -> None:
+ """Second add() with same name+desc within window returns None."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ result1 = pm.add("My Note", "body content", description="important")
+ assert result1 is not None
+ assert isinstance(result1, Path)
+
+ result2 = pm.add("My Note", "body content", description="important")
+ assert result2 is None
+
+ def test_duplicate_add_does_not_create_second_file(self, tmp_path: Path) -> None:
+ """Blocked duplicate does not write a second .md file."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ pm.add("Unique Entry", "content", description="desc")
+ # Count .md files (excluding MEMORY.md)
+ md_files = [f for f in tmp_path.glob("*.md") if f.name != "MEMORY.md"]
+ assert len(md_files) == 1
+
+ pm.add("Unique Entry", "content", description="desc")
+ md_files = [f for f in tmp_path.glob("*.md") if f.name != "MEMORY.md"]
+ # Still only 1 file
+ assert len(md_files) == 1
+
+ def test_different_entries_both_written(self, tmp_path: Path) -> None:
+ """Entries with different content are both persisted."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+ r1 = pm.add("First", "body1", description="desc1")
+ r2 = pm.add("Second", "body2", description="desc2")
+ assert r1 is not None
+ assert r2 is not None
+ assert r1 != r2
+
+ def test_add_after_window_expires(self, tmp_path: Path) -> None:
+ """Same content can be written again after dedup window expires."""
+ pm = PersistentMemory(memory_dir=tmp_path)
+
+ with patch("src.memory.persistent._time.time", return_value=1000.0):
+ r1 = pm.add("Retry Note", "body", description="retry desc")
+ assert r1 is not None
+
+ # Within window: blocked
+ with patch("src.memory.persistent._time.time", return_value=1010.0):
+ r2 = pm.add("Retry Note", "body", description="retry desc")
+ assert r2 is None
+
+ # After window: allowed
+ with patch("src.memory.persistent._time.time", return_value=1031.0):
+ r3 = pm.add("Retry Note", "body", description="retry desc")
+ assert r3 is not None
diff --git a/agent/tests/test_hotness_decay.py b/agent/tests/test_hotness_decay.py
new file mode 100644
index 000000000..a4e8e88be
--- /dev/null
+++ b/agent/tests/test_hotness_decay.py
@@ -0,0 +1,129 @@
+"""Tests for compute_hotness decay formula and feature flag switching."""
+
+from __future__ import annotations
+
+import math
+from pathlib import Path
+
+import pytest
+
+from src.config.accessor import reset_env_config
+from src.memory.persistent import PersistentMemory, compute_hotness
+
+
[email protected](autouse=True)
+def _reset_config_cache():
+ """Reset env config singleton so monkeypatch.setenv() takes effect."""
+ reset_env_config()
+ yield
+ reset_env_config()
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for compute_hotness formula
+# ---------------------------------------------------------------------------
+
+
+class TestComputeHotness:
+ """Validate compute_hotness mathematical properties."""
+
+ def test_zero_access_zero_days_yields_half(self) -> None:
+ """sigmoid(log1p(0)) = sigmoid(0) = 0.5; exp(0) = 1 => result ~0.5."""
+ result = compute_hotness(0, 0.0)
+ assert math.isclose(result, 0.5, rel_tol=1e-9)
+
+ def test_higher_access_count_increases_hotness(self) -> None:
+ """More accesses should increase the frequency signal."""
+ low = compute_hotness(0, 0.0)
+ high = compute_hotness(10, 0.0)
+ assert high > low
+
+ def test_seven_day_half_life(self) -> None:
+ """After 7 days, recency signal should halve."""
+ at_zero = compute_hotness(10, 0.0)
+ at_seven = compute_hotness(10, 7.0)
+ assert math.isclose(at_seven, at_zero * 0.5, rel_tol=1e-9)
+
+ def test_sigmoid_prevents_explosion(self) -> None:
+ """Even with extreme access_count, result stays below 1.0."""
+ result = compute_hotness(1000, 0.0)
+ assert result < 1.0
+
+ def test_monotone_decay_with_days(self) -> None:
+ """Hotness should monotonically decrease as days increase."""
+ values = [compute_hotness(5, d) for d in range(0, 30)]
+ for i in range(1, len(values)):
+ assert values[i] <= values[i - 1]
+
+ def test_large_days_approaches_zero(self) -> None:
+ """After many days, hotness should approach zero."""
+ result = compute_hotness(100, 365.0)
+ assert result < 0.01
+
+
+# ---------------------------------------------------------------------------
+# Feature flag integration test
+# ---------------------------------------------------------------------------
+
+
+def _create_memory_file(
+ tmp_path: Path,
+ name: str,
+ access_count: int = 5,
+ quality_score: float = 0.8,
+ last_accessed: str = "2020-01-01T00:00:00",
+) -> Path:
+ """Create a minimal memory file for testing."""
+ slug = name.lower().replace(" ", "_")[:40]
+ filename = f"project_{slug}.md"
+ path = tmp_path / filename
+ frontmatter = (
+ f"---\n"
+ f"name: {name}\n"
+ f"description: {name}\n"
+ f"type: project\n"
+ f"id: aabb11\n"
+ f"created_at: 2020-01-01T00:00:00\n"
+ f"updated_at: 2020-01-01T00:00:00\n"
+ f"keywords: []\n"
+ f"quality_score: {quality_score}\n"
+ f"access_count: {access_count}\n"
+ f"last_accessed: {last_accessed}\n"
+ f"importance: 0.5\n"
+ f"related_memories: []\n"
+ f"---\n\n"
+ f"test body for {name}\n"
+ )
+ path.write_text(frontmatter, encoding="utf-8")
+ return path
+
+
+class TestFeatureFlagSwitching:
+ """Verify that VT_MEMORY_HOTNESS_DECAY flag changes _scan_entries formula."""
+
+ def test_default_uses_ebbinghaus(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """With no flags set, importance uses compute_importance (returns quality_score as-is)."""
+ monkeypatch.delenv("VT_MEMORY_HOTNESS_DECAY", raising=False)
+ monkeypatch.delenv("VT_MEMORY_DECAY", raising=False)
+ reset_env_config()
+ _create_memory_file(tmp_path, "test_entry")
+ pm = PersistentMemory(memory_dir=tmp_path)
+ entries = pm.list_entries()
+ assert len(entries) == 1
+ # With decay disabled, compute_importance returns quality_score directly
+ assert entries[0].importance == 0.8
+
+ def test_hotness_flag_changes_formula(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ """When VT_MEMORY_HOTNESS_DECAY=1, _scan_entries uses compute_hotness."""
+ monkeypatch.setenv("VT_MEMORY_HOTNESS_DECAY", "1")
+ reset_env_config()
+ _create_memory_file(tmp_path, "test_entry", access_count=5)
+ pm = PersistentMemory(memory_dir=tmp_path)
+ entries = pm.list_entries()
+ assert len(entries) == 1
+ # compute_hotness for access_count=5 at many days since access
+ # should be different from plain quality_score=0.8
+ importance = entries[0].importance
+ assert importance != 0.8
+ # Should be positive and less than 1
+ assert 0.0 < importance < 1.0
diff --git a/bench_report.json b/bench_report.json
new file mode 100644
index 000000000..6dabbbdd7
--- /dev/null
+++ b/bench_report.json
@@ -0,0 +1,35 @@
+{
+ "timestamp": "2026-07-21T06:04:45",
+ "corpus_size": 200,
+ "query_count": 50,
+ "baseline": {
+ "p_at_5": 0.372,
+ "mrr": 0.5643,
+ "ndcg_at_5": 0.3894
+ },
+ "treatment": {
+ "p_at_5": 0.42,
+ "mrr": 0.6227,
+ "ndcg_at_5": 0.4402
+ },
+ "improvement": {
+ "p_at_5_relative": "+12.90%",
+ "mrr_relative": "+10.34%",
+ "ndcg_at_5_relative": "+13.06%"
+ },
+ "by_difficulty": {
+ "easy": {
+ "baseline_p5": 0.52,
+ "treatment_p5": 0.61
+ },
+ "medium": {
+ "baseline_p5": 0.33,
+ "treatment_p5": 0.33
+ },
+ "hard": {
+ "baseline_p5": 0.16,
+ "treatment_p5": 0.22
+ }
+ },
+ "gate_passed": true
+}
\ No newline at end of file