Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
feat(evals): weekly HotpotQA answer-recall report on the prose path (#1188)
Ashish46d4378cf779
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.ymlindex 11dcc124b..92bd0c159 100644--- a/.github/workflows/eval.yml+++ b/.github/workflows/eval.yml@@ -144,6 +144,32 @@ jobs:print(f'::warning title=Fidelity recall::{result.failed_cases} case(s) fell below 0.9 recall: {result.errors[:3]}')"+ # Real-dataset recall on the prose path (HotpotQA): does the ground-truth+ # answer survive compressing the supporting context? Uses the production+ # routing path, so prose flows through Kompress (ModernBERT) — allowed here+ # because the weekly job installs [all]. Non-blocking and defensive: a+ # dataset download or model failure warns rather than fails the job.+ - name: Dataset recall report — HotpotQA (model-allowed, non-blocking)+ run: |+ python -c "+ try:+ from headroom.transforms.kompress_compressor import warm_kompress_model+ from headroom.evals.datasets import load_hotpotqa+ from headroom.evals.runners.compression_only import CompressionOnlyRunner+ # Block until the Kompress model is loaded; otherwise prose passes+ # through uncompressed and the recall number is meaningless.+ warmed = warm_kompress_model()+ suite = load_hotpotqa(n=50)+ result = CompressionOnlyRunner().evaluate_dataset_recall(suite)+ print(f'HotpotQA answer recall: {result.passed_cases}/{result.total_cases} probeable cases >=0.9, avg compression {result.avg_compression_ratio:.1%} (model_warmed={warmed})')+ if result.avg_compression_ratio < 0.01:+ print('::warning title=Dataset recall::compression did not engage (~0%); recall is not a meaningful fidelity signal — check Kompress model availability')+ elif result.failed_cases:+ print(f'::warning title=Dataset recall::{result.failed_cases} HotpotQA case(s) lost the answer under compression')+ except Exception as e:+ print(f'::warning title=Dataset recall::skipped (dataset/model unavailable): {e}')+ " || true+- name: Upload resultsif: always()uses: actions/upload-artifact@v7diff --git a/headroom/evals/runners/compression_only.py b/headroom/evals/runners/compression_only.pyindex e3ad17888..51a8a8f3d 100644--- a/headroom/evals/runners/compression_only.py+++ b/headroom/evals/runners/compression_only.py@@ -14,7 +14,10 @@ import jsonimport loggingimport timefrom dataclasses import dataclass, field-from typing import Any+from typing import TYPE_CHECKING, Any++if TYPE_CHECKING:+ from headroom.evals.core import EvalSuitelogger = logging.getLogger(__name__)@@ -242,6 +245,91 @@ class CompressionOnlyRunner:errors=errors,)+ def evaluate_dataset_recall(+ self,+ suite: EvalSuite,+ recall_threshold: float = 0.9,+ min_answer_chars: int = 4,+ ) -> CompressionOnlyResult:+ """Compress each QA case's context and check its answer survives.++ The probe is the case's ``ground_truth`` answer. A case only counts when+ the answer literally appears in the original context (otherwise survival+ is not measurable); trivial answers (too short, or yes/no) are skipped.+ Compression uses the production routing path, so prose flows through+ Kompress (ModernBERT) — intended for the model-allowed weekly job, not+ the hermetic per-PR gate.++ Args:+ suite: An EvalSuite of QA cases (e.g. from ``load_hotpotqa``).+ recall_threshold: Minimum answer recall for a case to pass.+ min_answer_chars: Answers shorter than this are skipped as un-probeable.+ """+ from headroom.evals.metrics import compute_information_recall+ from headroom.transforms.content_router import ContentRouter++ trivial = {"yes", "no", "true", "false"}+ start_time = time.time()+ router = ContentRouter()+ passed = 0+ failed = 0+ total_original = 0+ total_compressed = 0+ details: list[dict[str, Any]] = []+ errors: list[str] = []++ for case in suite.cases:+ answer = (case.ground_truth or "").strip()+ if len(answer) < min_answer_chars or answer.lower() in trivial:+ continue # un-probeable: survival of this answer carries no signal+ if answer.lower() not in case.context.lower():+ continue # answer not literally in context; nothing to measure++ original_tokens = self._estimate_tokens(case.context)+ try:+ compressed = router.compress(case.context).compressed+ compressed_tokens = self._estimate_tokens(compressed)+ recall = compute_information_recall(case.context, compressed, [answer])["recall"]+ is_pass = recall >= recall_threshold++ total_original += original_tokens+ total_compressed += compressed_tokens+ passed += is_pass+ failed += not is_pass+ details.append(+ {+ "id": case.id,+ "passed": is_pass,+ "recall": recall,+ "answer": answer,+ "compression_ratio": 1 - (compressed_tokens / original_tokens)+ if original_tokens > 0+ else 0,+ }+ )+ except Exception as e:+ failed += 1+ errors.append(f"Dataset recall error for {case.id}: {e}")+ details.append({"id": case.id, "passed": False, "error": str(e)})++ total_cases = passed + failed+ ratios = [d["compression_ratio"] for d in details if "compression_ratio" in d]++ return CompressionOnlyResult(+ benchmark=f"dataset_recall:{suite.name}",+ total_cases=total_cases,+ passed_cases=passed,+ failed_cases=failed,+ accuracy_rate=passed / total_cases if total_cases > 0 else 0.0,+ avg_compression_ratio=sum(ratios) / len(ratios) if ratios else 0.0,+ total_original_tokens=total_original,+ total_compressed_tokens=total_compressed,+ total_tokens_saved=total_original - total_compressed,+ duration_seconds=time.time() - start_time,+ details=details,+ errors=errors,+ )+def generate_ccr_test_cases(self, n: int = 50) -> list[dict[str, Any]]:"""Generate synthetic test cases for CCR needle-retention testing.diff --git a/headroom/transforms/kompress_compressor.py b/headroom/transforms/kompress_compressor.pyindex fc9b6d212..596ec1b44 100644--- a/headroom/transforms/kompress_compressor.py+++ b/headroom/transforms/kompress_compressor.py@@ -946,6 +946,33 @@ def ensure_background_download(model_id: str = HF_MODEL_ID, device: str = "auto"thread.start()+def warm_kompress_model(+ model_id: str = HF_MODEL_ID,+ device: str = "cpu",+ *,+ allow_download: bool = True,+) -> bool:+ """Synchronously load the Kompress model, blocking until it is ready.++ Unlike :func:`ensure_background_download` (which loads in a daemon thread and+ lets early requests pass through uncompressed while the download runs), this+ blocks the caller so the *next* compression uses the model rather than+ passing through. Intended for batch/eval contexts that must measure real+ compression, not passthrough.++ Returns ``True`` if the model is loaded and ready, ``False`` if Kompress is+ unavailable or the load failed.+ """+ if not is_kompress_available():+ return False+ try:+ _load_kompress(model_id, device, allow_download=allow_download)+ return model_id in _kompress_cache+ except Exception as exc: # pragma: no cover - network/model load failure+ logger.warning("Kompress: synchronous warm failed for %s: %s", model_id, exc)+ return False++# ── Compressor ────────────────────────────────────────────────────────diff --git a/tests/test_dataset_recall_runner.py b/tests/test_dataset_recall_runner.pynew file mode 100644index 000000000..3b205b762--- /dev/null+++ b/tests/test_dataset_recall_runner.py@@ -0,0 +1,119 @@+"""Hermetic unit tests for CompressionOnlyRunner.evaluate_dataset_recall.++Exercises the dataset-recall plumbing with synthetic JSON-array contexts (which+route through SmartCrusher / Rust — no model, no network) so it runs in the+standard [dev] shard. The weekly job drives the same method with real prose+datasets (HotpotQA), which is intentionally not exercised here.+"""++from __future__ import annotations++import json++from headroom.evals.core import EvalCase, EvalSuite+from headroom.evals.runners.compression_only import CompressionOnlyRunner+++def _array_context_with(answer: str) -> str:+ """A JSON-array tool output whose error row embeds ``answer`` (a kept row)."""+ rows = [{"seq": i, "level": "INFO", "status": "ok", "msg": f"heartbeat {i}"} for i in range(30)]+ rows[14] = {"seq": 14, "level": "ERROR", "status": "failed", "msg": answer}+ return json.dumps(rows)+++def _suite() -> EvalSuite:+ answer = "PaymentService NullPointerException at charge line 88"+ return EvalSuite(+ name="synthetic",+ cases=[+ # Probeable: answer is in an error row -> retained -> recall 1.0.+ EvalCase(+ id="probeable",+ context=_array_context_with(answer),+ query="what failed?",+ ground_truth=answer,+ ),+ # Skipped: trivial yes/no answer.+ EvalCase(+ id="trivial",+ context=_array_context_with(answer),+ query="did it fail?",+ ground_truth="yes",+ ),+ # Skipped: answer not present in the context at all.+ EvalCase(+ id="absent",+ context=_array_context_with(answer),+ query="?",+ ground_truth="totally-absent-token-xyz",+ ),+ ],+ )+++def test_dataset_recall_counts_only_probeable_cases() -> None:+ result = CompressionOnlyRunner().evaluate_dataset_recall(_suite())+ # Only the "probeable" case is measurable; trivial + absent are skipped.+ assert result.total_cases == 1+ assert result.passed_cases == 1+ assert result.accuracy_rate == 1.0+ assert result.benchmark == "dataset_recall:synthetic"+++def test_dataset_recall_empty_suite_is_safe() -> None:+ result = CompressionOnlyRunner().evaluate_dataset_recall(EvalSuite(name="empty", cases=[]))+ assert result.total_cases == 0+ assert result.accuracy_rate == 0.0+ assert result.errors == []+++def test_dataset_recall_records_compression_errors(monkeypatch) -> None:+ # A compressor crash on one case must not abort the run: the case counts+ # as failed, the error is recorded, and the detail row carries it.+ from headroom.transforms.content_router import ContentRouter++ def _boom(self, content, context="", question=None, bias=1.0):+ raise RuntimeError("router exploded")++ monkeypatch.setattr(ContentRouter, "compress", _boom)+ result = CompressionOnlyRunner().evaluate_dataset_recall(_suite())+ assert result.total_cases == 1+ assert result.failed_cases == 1+ assert result.passed_cases == 0+ assert result.errors and "router exploded" in result.errors[0]+ assert result.details[0]["passed"] is False+ assert "router exploded" in result.details[0]["error"]+++def test_warm_kompress_model_returns_false_when_unavailable(monkeypatch) -> None:+ # Guard path: no Kompress backend -> no download attempt, returns False.+ import headroom.transforms.kompress_compressor as kc++ monkeypatch.setattr(kc, "is_kompress_available", lambda: False)+ assert kc.warm_kompress_model() is False+++def test_warm_kompress_model_true_when_load_populates_cache(monkeypatch) -> None:+ # Success path: the synchronous load lands the model in the cache.+ import headroom.transforms.kompress_compressor as kc++ cache: dict[str, object] = {}+ monkeypatch.setattr(kc, "_kompress_cache", cache)+ monkeypatch.setattr(kc, "is_kompress_available", lambda: True)+ monkeypatch.setattr(+ kc,+ "_load_kompress",+ lambda model_id, device, allow_download: cache.setdefault(model_id, object()),+ )+ assert kc.warm_kompress_model("test-model") is True+++def test_warm_kompress_model_false_when_load_leaves_cache_empty(monkeypatch) -> None:+ # The loader returned without raising but the model never landed in the+ # cache (e.g. download disallowed and not cached locally).+ import headroom.transforms.kompress_compressor as kc++ monkeypatch.setattr(kc, "_kompress_cache", {})+ monkeypatch.setattr(kc, "is_kompress_available", lambda: True)+ monkeypatch.setattr(kc, "_load_kompress", lambda model_id, device, allow_download: None)+ assert kc.warm_kompress_model("test-model", allow_download=False) is False