Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
test(evals): add offline fidelity regression gate (recall-based, zero-model) (#1187)
Ashish23d73ae0708b
diff --git a/.github/workflows/eval.yml b/.github/workflows/eval.ymlindex 3037d6c2b..f1c031ca6 100644--- a/.github/workflows/eval.yml+++ b/.github/workflows/eval.yml@@ -126,6 +126,24 @@ jobs:env:OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}++ # Recall-based fidelity report on the production routing path. Zero cost+ # (synthetic structured cases -> Rust compressors; no model, no API, no+ # secrets). Non-blocking: surfaces recall trends weekly without gating.+ # The blocking per-PR fidelity gate lives in+ # tests/test_compression_fidelity_regression.py (runs in the [dev] shard).+ - name: Information-retention recall report (zero cost, non-blocking)+ run: |+ python -c "+ from headroom.evals.runners.compression_only import CompressionOnlyRunner+ runner = CompressionOnlyRunner()+ cases = runner.generate_info_retention_cases(n=50)+ result = runner.evaluate_information_retention(cases)+ print(f'Information retention: {result.passed_cases}/{result.total_cases} cases >=0.9 recall, avg compression {result.avg_compression_ratio:.1%}')+ if not result.passed:+ print(f'::warning title=Fidelity recall::{result.failed_cases} case(s) fell below 0.9 recall: {result.errors[:3]}')+ "+- name: Upload resultsif: always()uses: actions/upload-artifact@v7diff --git a/tests/fixtures/fidelity_golden/_generate.py b/tests/fixtures/fidelity_golden/_generate.pynew file mode 100644index 000000000..9ec25ef82--- /dev/null+++ b/tests/fixtures/fidelity_golden/_generate.py@@ -0,0 +1,141 @@+"""Deterministic generator for the fidelity-regression golden fixtures.++Run to (re)create ``cases.json``::++ python tests/fixtures/fidelity_golden/_generate.py++Each case is a JSON tool output (a list of records) plus the question it should+let a model answer and the evidence that must survive compression. Critical+``answer_evidence`` is always placed in an *error* or *anomaly* row, because+those are the rows SmartCrusher formally guarantees to retain+(see ``tests/test_quality_retention.py``) — so the hard ``recall == 1.0`` gate+holds on a healthy tree. ``supporting_facts`` are softer details that may live+in droppable rows; they feed the aggregate-recall baseline, not the hard gate.++No models, no network: content is built from fixed strings only.+"""++from __future__ import annotations++import json+from pathlib import Path++OUT = Path(__file__).parent / "cases.json"+++def _filler(n: int, msg: str) -> list[dict]:+ """n benign, near-identical INFO rows (droppable noise)."""+ return [{"seq": i, "level": "INFO", "status": "ok", "msg": f"{msg} {i}"} for i in range(n)]+++def _cases() -> list[dict]:+ cases: list[dict] = []++ # 1. OOM crash buried in benign logs — error-keyword retention.+ rows = _filler(28, "heartbeat ping")+ rows.insert(+ 17,+ {+ "seq": 17,+ "level": "ERROR",+ "status": "failed",+ "msg": "OOM killed worker 3 exceeded memory limit 8GiB",+ },+ )+ rows.insert(+ 9, {"seq": 9, "level": "INFO", "status": "ok", "msg": "checkpoint saved at step 4500"}+ )+ cases.append(+ {+ "id": "logs_oom",+ "question": "Why did the job fail?",+ "content_type": "json_array",+ "compress": {"with_compaction": False, "max_items_after_crush": 10},+ "answer_evidence": ["OOM killed worker 3", "exceeded memory limit"],+ "supporting_facts": ["checkpoint saved at step 4500"],+ "content": rows,+ }+ )++ # 2. Payment exception — error-keyword retention.+ rows = _filler(30, "GET /healthz 200")+ rows.insert(+ 21,+ {+ "seq": 21,+ "level": "ERROR",+ "status": "failed",+ "msg": "exception: NullPointerException at PaymentService.charge line 88",+ },+ )+ cases.append(+ {+ "id": "payment_exception",+ "question": "Which service threw an exception and where?",+ "content_type": "json_array",+ "compress": {"with_compaction": False, "max_items_after_crush": 8},+ "answer_evidence": ["NullPointerException", "PaymentService.charge line 88"],+ "supporting_facts": [],+ "content": rows,+ }+ )++ # 3. Latency anomaly — anomaly (>2 sigma) retention. Values cluster near 100ms,+ # one row spikes to 99999ms.+ rows = [+ {"seq": i, "region": "us-west-2", "latency_ms": 95 + (i % 11), "status": "ok"}+ for i in range(30)+ ]+ rows[19] = {+ "seq": 19,+ "region": "us-east-1",+ "latency_ms": 99999,+ "status": "ok",+ "note": "latency spike us-east-1",+ }+ cases.append(+ {+ "id": "latency_anomaly",+ "question": "Which region had the latency spike, and how high?",+ "content_type": "json_array",+ "compress": {"with_compaction": False, "max_items_after_crush": 8},+ "answer_evidence": ["us-east-1", "99999"],+ "supporting_facts": [],+ "content": rows,+ }+ )++ # 4. CI test failure among many passes — "failed" keyword retention.+ rows = [+ {"seq": i, "test": f"test_module_{i}", "outcome": "passed", "duration_ms": 5 + i}+ for i in range(30)+ ]+ rows[12] = {+ "seq": 12,+ "test": "test_auth_token_refresh",+ "outcome": "failed",+ "duration_ms": 41,+ "error": "AssertionError: expected 200 got 401 in test_auth_token_refresh",+ }+ cases.append(+ {+ "id": "ci_test_failures",+ "question": "Which test failed and why?",+ "content_type": "json_array",+ "compress": {"with_compaction": False, "max_items_after_crush": 9},+ "answer_evidence": ["test_auth_token_refresh", "expected 200 got 401"],+ "supporting_facts": [],+ "content": rows,+ }+ )++ return cases+++def main() -> None:+ OUT.write_text(json.dumps(_cases(), indent=2) + "\n")+ print(f"wrote {OUT} ({len(_cases())} cases)")+++if __name__ == "__main__":+ main()diff --git a/tests/fixtures/fidelity_golden/baseline.json b/tests/fixtures/fidelity_golden/baseline.jsonnew file mode 100644index 000000000..44f0b7023--- /dev/null+++ b/tests/fixtures/fidelity_golden/baseline.json@@ -0,0 +1,5 @@+{+ "aggregate_recall": 0.9167,+ "tolerance": 0.02,+ "note": "Mean information-recall across all golden cases (answer_evidence + supporting_facts) after lossy SmartCrusher compression. Deterministic. Regenerate fixtures via `python tests/fixtures/fidelity_golden/_generate.py`; update this value only when a recall change is intended and understood."+}diff --git a/tests/fixtures/fidelity_golden/cases.json b/tests/fixtures/fidelity_golden/cases.jsonnew file mode 100644index 000000000..13c1dc905--- /dev/null+++ b/tests/fixtures/fidelity_golden/cases.json@@ -0,0 +1,796 @@+[+ {+ "id": "logs_oom",+ "question": "Why did the job fail?",+ "content_type": "json_array",+ "compress": {+ "with_compaction": false,+ "max_items_after_crush": 10+ },+ "answer_evidence": [+ "OOM killed worker 3",+ "exceeded memory limit"+ ],+ "supporting_facts": [+ "checkpoint saved at step 4500"+ ],+ "content": [+ {+ "seq": 0,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 0"+ },+ {+ "seq": 1,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 1"+ },+ {+ "seq": 2,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 2"+ },+ {+ "seq": 3,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 3"+ },+ {+ "seq": 4,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 4"+ },+ {+ "seq": 5,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 5"+ },+ {+ "seq": 6,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 6"+ },+ {+ "seq": 7,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 7"+ },+ {+ "seq": 8,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 8"+ },+ {+ "seq": 9,+ "level": "INFO",+ "status": "ok",+ "msg": "checkpoint saved at step 4500"+ },+ {+ "seq": 9,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 9"+ },+ {+ "seq": 10,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 10"+ },+ {+ "seq": 11,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 11"+ },+ {+ "seq": 12,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 12"+ },+ {+ "seq": 13,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 13"+ },+ {+ "seq": 14,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 14"+ },+ {+ "seq": 15,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 15"+ },+ {+ "seq": 16,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 16"+ },+ {+ "seq": 17,+ "level": "ERROR",+ "status": "failed",+ "msg": "OOM killed worker 3 exceeded memory limit 8GiB"+ },+ {+ "seq": 17,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 17"+ },+ {+ "seq": 18,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 18"+ },+ {+ "seq": 19,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 19"+ },+ {+ "seq": 20,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 20"+ },+ {+ "seq": 21,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 21"+ },+ {+ "seq": 22,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 22"+ },+ {+ "seq": 23,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 23"+ },+ {+ "seq": 24,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 24"+ },+ {+ "seq": 25,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 25"+ },+ {+ "seq": 26,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 26"+ },+ {+ "seq": 27,+ "level": "INFO",+ "status": "ok",+ "msg": "heartbeat ping 27"+ }+ ]+ },+ {+ "id": "payment_exception",+ "question": "Which service threw an exception and where?",+ "content_type": "json_array",+ "compress": {+ "with_compaction": false,+ "max_items_after_crush": 8+ },+ "answer_evidence": [+ "NullPointerException",+ "PaymentService.charge line 88"+ ],+ "supporting_facts": [],+ "content": [+ {+ "seq": 0,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 0"+ },+ {+ "seq": 1,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 1"+ },+ {+ "seq": 2,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 2"+ },+ {+ "seq": 3,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 3"+ },+ {+ "seq": 4,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 4"+ },+ {+ "seq": 5,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 5"+ },+ {+ "seq": 6,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 6"+ },+ {+ "seq": 7,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 7"+ },+ {+ "seq": 8,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 8"+ },+ {+ "seq": 9,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 9"+ },+ {+ "seq": 10,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 10"+ },+ {+ "seq": 11,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 11"+ },+ {+ "seq": 12,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 12"+ },+ {+ "seq": 13,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 13"+ },+ {+ "seq": 14,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 14"+ },+ {+ "seq": 15,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 15"+ },+ {+ "seq": 16,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 16"+ },+ {+ "seq": 17,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 17"+ },+ {+ "seq": 18,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 18"+ },+ {+ "seq": 19,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 19"+ },+ {+ "seq": 20,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 20"+ },+ {+ "seq": 21,+ "level": "ERROR",+ "status": "failed",+ "msg": "exception: NullPointerException at PaymentService.charge line 88"+ },+ {+ "seq": 21,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 21"+ },+ {+ "seq": 22,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 22"+ },+ {+ "seq": 23,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 23"+ },+ {+ "seq": 24,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 24"+ },+ {+ "seq": 25,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 25"+ },+ {+ "seq": 26,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 26"+ },+ {+ "seq": 27,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 27"+ },+ {+ "seq": 28,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 28"+ },+ {+ "seq": 29,+ "level": "INFO",+ "status": "ok",+ "msg": "GET /healthz 200 29"+ }+ ]+ },+ {+ "id": "latency_anomaly",+ "question": "Which region had the latency spike, and how high?",+ "content_type": "json_array",+ "compress": {+ "with_compaction": false,+ "max_items_after_crush": 8+ },+ "answer_evidence": [+ "us-east-1",+ "99999"+ ],+ "supporting_facts": [],+ "content": [+ {+ "seq": 0,+ "region": "us-west-2",+ "latency_ms": 95,+ "status": "ok"+ },+ {+ "seq": 1,+ "region": "us-west-2",+ "latency_ms": 96,+ "status": "ok"+ },+ {+ "seq": 2,+ "region": "us-west-2",+ "latency_ms": 97,+ "status": "ok"+ },+ {+ "seq": 3,+ "region": "us-west-2",+ "latency_ms": 98,+ "status": "ok"+ },+ {+ "seq": 4,+ "region": "us-west-2",+ "latency_ms": 99,+ "status": "ok"+ },+ {+ "seq": 5,+ "region": "us-west-2",+ "latency_ms": 100,+ "status": "ok"+ },+ {+ "seq": 6,+ "region": "us-west-2",+ "latency_ms": 101,+ "status": "ok"+ },+ {+ "seq": 7,+ "region": "us-west-2",+ "latency_ms": 102,+ "status": "ok"+ },+ {+ "seq": 8,+ "region": "us-west-2",+ "latency_ms": 103,+ "status": "ok"+ },+ {+ "seq": 9,+ "region": "us-west-2",+ "latency_ms": 104,+ "status": "ok"+ },+ {+ "seq": 10,+ "region": "us-west-2",+ "latency_ms": 105,+ "status": "ok"+ },+ {+ "seq": 11,+ "region": "us-west-2",+ "latency_ms": 95,+ "status": "ok"+ },+ {+ "seq": 12,+ "region": "us-west-2",+ "latency_ms": 96,+ "status": "ok"+ },+ {+ "seq": 13,+ "region": "us-west-2",+ "latency_ms": 97,+ "status": "ok"+ },+ {+ "seq": 14,+ "region": "us-west-2",+ "latency_ms": 98,+ "status": "ok"+ },+ {+ "seq": 15,+ "region": "us-west-2",+ "latency_ms": 99,+ "status": "ok"+ },+ {+ "seq": 16,+ "region": "us-west-2",+ "latency_ms": 100,+ "status": "ok"+ },+ {+ "seq": 17,+ "region": "us-west-2",+ "latency_ms": 101,+ "status": "ok"+ },+ {+ "seq": 18,+ "region": "us-west-2",+ "latency_ms": 102,+ "status": "ok"+ },+ {+ "seq": 19,+ "region": "us-east-1",+ "latency_ms": 99999,+ "status": "ok",+ "note": "latency spike us-east-1"+ },+ {+ "seq": 20,+ "region": "us-west-2",+ "latency_ms": 104,+ "status": "ok"+ },+ {+ "seq": 21,+ "region": "us-west-2",+ "latency_ms": 105,+ "status": "ok"+ },+ {+ "seq": 22,+ "region": "us-west-2",+ "latency_ms": 95,+ "status": "ok"+ },+ {+ "seq": 23,+ "region": "us-west-2",+ "latency_ms": 96,+ "status": "ok"+ },+ {+ "seq": 24,+ "region": "us-west-2",+ "latency_ms": 97,+ "status": "ok"+ },+ {+ "seq": 25,+ "region": "us-west-2",+ "latency_ms": 98,+ "status": "ok"+ },+ {+ "seq": 26,+ "region": "us-west-2",+ "latency_ms": 99,+ "status": "ok"+ },+ {+ "seq": 27,+ "region": "us-west-2",+ "latency_ms": 100,+ "status": "ok"+ },+ {+ "seq": 28,+ "region": "us-west-2",+ "latency_ms": 101,+ "status": "ok"+ },+ {+ "seq": 29,+ "region": "us-west-2",+ "latency_ms": 102,+ "status": "ok"+ }+ ]+ },+ {+ "id": "ci_test_failures",+ "question": "Which test failed and why?",+ "content_type": "json_array",+ "compress": {+ "with_compaction": false,+ "max_items_after_crush": 9+ },+ "answer_evidence": [+ "test_auth_token_refresh",+ "expected 200 got 401"+ ],+ "supporting_facts": [],+ "content": [+ {+ "seq": 0,+ "test": "test_module_0",+ "outcome": "passed",+ "duration_ms": 5+ },+ {+ "seq": 1,+ "test": "test_module_1",+ "outcome": "passed",+ "duration_ms": 6+ },+ {+ "seq": 2,+ "test": "test_module_2",+ "outcome": "passed",+ "duration_ms": 7+ },+ {+ "seq": 3,+ "test": "test_module_3",+ "outcome": "passed",+ "duration_ms": 8+ },+ {+ "seq": 4,+ "test": "test_module_4",+ "outcome": "passed",+ "duration_ms": 9+ },+ {+ "seq": 5,+ "test": "test_module_5",+ "outcome": "passed",+ "duration_ms": 10+ },+ {+ "seq": 6,+ "test": "test_module_6",+ "outcome": "passed",+ "duration_ms": 11+ },+ {+ "seq": 7,+ "test": "test_module_7",+ "outcome": "passed",+ "duration_ms": 12+ },+ {+ "seq": 8,+ "test": "test_module_8",+ "outcome": "passed",+ "duration_ms": 13+ },+ {+ "seq": 9,+ "test": "test_module_9",+ "outcome": "passed",+ "duration_ms": 14+ },+ {+ "seq": 10,+ "test": "test_module_10",+ "outcome": "passed",+ "duration_ms": 15+ },+ {+ "seq": 11,+ "test": "test_module_11",+ "outcome": "passed",+ "duration_ms": 16+ },+ {+ "seq": 12,+ "test": "test_auth_token_refresh",+ "outcome": "failed",+ "duration_ms": 41,+ "error": "AssertionError: expected 200 got 401 in test_auth_token_refresh"+ },+ {+ "seq": 13,+ "test": "test_module_13",+ "outcome": "passed",+ "duration_ms": 18+ },+ {+ "seq": 14,+ "test": "test_module_14",+ "outcome": "passed",+ "duration_ms": 19+ },+ {+ "seq": 15,+ "test": "test_module_15",+ "outcome": "passed",+ "duration_ms": 20+ },+ {+ "seq": 16,+ "test": "test_module_16",+ "outcome": "passed",+ "duration_ms": 21+ },+ {+ "seq": 17,+ "test": "test_module_17",+ "outcome": "passed",+ "duration_ms": 22+ },+ {+ "seq": 18,+ "test": "test_module_18",+ "outcome": "passed",+ "duration_ms": 23+ },+ {+ "seq": 19,+ "test": "test_module_19",+ "outcome": "passed",+ "duration_ms": 24+ },+ {+ "seq": 20,+ "test": "test_module_20",+ "outcome": "passed",+ "duration_ms": 25+ },+ {+ "seq": 21,+ "test": "test_module_21",+ "outcome": "passed",+ "duration_ms": 26+ },+ {+ "seq": 22,+ "test": "test_module_22",+ "outcome": "passed",+ "duration_ms": 27+ },+ {+ "seq": 23,+ "test": "test_module_23",+ "outcome": "passed",+ "duration_ms": 28+ },+ {+ "seq": 24,+ "test": "test_module_24",+ "outcome": "passed",+ "duration_ms": 29+ },+ {+ "seq": 25,+ "test": "test_module_25",+ "outcome": "passed",+ "duration_ms": 30+ },+ {+ "seq": 26,+ "test": "test_module_26",+ "outcome": "passed",+ "duration_ms": 31+ },+ {+ "seq": 27,+ "test": "test_module_27",+ "outcome": "passed",+ "duration_ms": 32+ },+ {+ "seq": 28,+ "test": "test_module_28",+ "outcome": "passed",+ "duration_ms": 33+ },+ {+ "seq": 29,+ "test": "test_module_29",+ "outcome": "passed",+ "duration_ms": 34+ }+ ]+ }+]diff --git a/tests/test_compression_fidelity_regression.py b/tests/test_compression_fidelity_regression.pynew file mode 100644index 000000000..a83cea13d--- /dev/null+++ b/tests/test_compression_fidelity_regression.py@@ -0,0 +1,72 @@+"""Offline fidelity regression gate (recall-based, zero-model).++Compresses vendored golden tool-output fixtures through SmartCrusher's lossy+path and asserts that the evidence a model needs to answer each case's question+survives compression. Scoring is pure stdlib (``headroom.evals.metrics``) — no+ML model, no network, no API keys — so this runs in the standard ``[dev]`` CI+shard as a blocking PR check.++A failure here means a code change made lossy compression silently drop+information that answers a known question. Fixtures and the committed baseline+are generated by ``tests/fixtures/fidelity_golden/_generate.py``.+"""++from __future__ import annotations++import json+from pathlib import Path++import pytest++from headroom.evals.metrics import compute_information_recall+from headroom.transforms.smart_crusher import SmartCrusherConfig, smart_crush_tool_output++FIXTURE_DIR = Path(__file__).parent / "fixtures" / "fidelity_golden"+CASES: list[dict] = json.loads((FIXTURE_DIR / "cases.json").read_text())+BASELINE: dict = json.loads((FIXTURE_DIR / "baseline.json").read_text())+++def _compress(case: dict) -> tuple[str, str]:+ """Compress a case's tool output via the lossy SmartCrusher path (no model)."""+ original = json.dumps(case["content"])+ cfg = SmartCrusherConfig(max_items_after_crush=case["compress"]["max_items_after_crush"])+ crushed, _modified, _info = smart_crush_tool_output(+ original, cfg, with_compaction=case["compress"]["with_compaction"]+ )+ return original, crushed++[email protected]("case", CASES, ids=[c["id"] for c in CASES])+def test_critical_evidence_survives_compression(case: dict) -> None:+ """Every ``answer_evidence`` string MUST survive lossy compression (recall == 1.0).++ Critical evidence lives in error/anomaly rows, which SmartCrusher formally+ guarantees to retain (see ``tests/test_quality_retention.py``).+ """+ original, crushed = _compress(case)+ result = compute_information_recall(original, crushed, case["answer_evidence"])+ assert result["recall"] == 1.0, (+ f"FIDELITY REGRESSION in '{case['id']}': compression dropped evidence "+ f"needed to answer {case['question']!r}. Lost: {result['facts_lost']}"+ )+++def test_aggregate_recall_not_regressed() -> None:+ """Mean recall over all evidence must not fall below the committed baseline.++ Catches softer regressions (e.g. relevant-but-non-critical context being+ dropped more aggressively) that the per-case critical gate would not.+ """+ recalls = []+ for case in CASES:+ original, crushed = _compress(case)+ probes = case["answer_evidence"] + case["supporting_facts"]+ recalls.append(compute_information_recall(original, crushed, probes)["recall"])++ mean_recall = sum(recalls) / len(recalls)+ floor = BASELINE["aggregate_recall"] - BASELINE["tolerance"]+ assert mean_recall >= floor, (+ f"FIDELITY REGRESSION: mean recall {mean_recall:.4f} fell below baseline "+ f"floor {floor:.4f} (baseline {BASELINE['aggregate_recall']} - tolerance "+ f"{BASELINE['tolerance']}). If this drop is intended, regenerate the baseline."+ )