Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

115 KB
Raw Blame
from __future__ import annotations

import json
import threading
from types import SimpleNamespace

import pytest

import headroom._ort as ort_runtime
import headroom.transforms.content_router as content_router_module
from headroom.transforms.content_detector import ContentType, DetectionResult
from headroom.transforms.content_router import (
    CompressionCache,
    CompressionStrategy,
    ContentRouter,
    ContentRouterConfig,
    RouterCompressionResult,
    RoutingDecision,
    _create_content_signature,
    _detect_content,
    _estimate_tokens,
    _extract_json_block,
    _strip_detection_envelope,
    is_mixed_content,
    split_into_sections,
)


@pytest.fixture(autouse=True)
def _reset_detect_module_state(monkeypatch: pytest.MonkeyPatch) -> None:
    """Keep the module-level detect flags from leaking across tests.

    The circuit breaker (#575) is process-wide, so a test that trips it would
    otherwise force later tests onto the pure-Python path. ``monkeypatch.setattr``
    zeroes each flag for the test and auto-restores it afterward.
    """
    monkeypatch.setattr(content_router_module, "_detect_native_unhealthy", False)
    monkeypatch.setattr(content_router_module, "_detect_backend_warned", False)
    monkeypatch.setattr(content_router_module, "_detect_panic_warned", False)
    # Router unit tests replace ``headroom._core.detect_content_type`` with
    # deterministic fakes. Keep the separate ORT API-compatibility preflight
    # open so those fakes reach the watchdog/circuit-breaker behavior under
    # test; incompatibility itself is covered in test_ort_dylib.py (#2960).
    monkeypatch.setattr(ort_runtime, "rust_ort_runtime_compatible", lambda: True)


def test_compression_cache_handles_hits_skips_evictions_and_clear(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    times = iter([100.0, 100.0, 100.0, 100.0, 100.0, 100.0, 112.0, 112.0])
    monkeypatch.setattr(content_router_module.time, "monotonic", lambda: next(times))
    monkeypatch.setattr(content_router_module.time, "perf_counter_ns", lambda: 50)

    cache = CompressionCache(ttl_seconds=10)
    cache.put(1, "compressed", 0.4, "text")
    cache.mark_skip(2)

    assert cache.get(1) == ("compressed", 0.4, "text")
    assert cache.is_skipped(2) is True
    assert cache.size == 1
    assert cache.skip_size == 1

    cache.move_to_skip(1)
    assert cache.get(1) is None
    assert cache.is_skipped(1) is True

    # Expire both skip entries
    assert cache.is_skipped(2) is False
    assert cache.is_skipped(1) is False

    assert cache.stats["cache_hits"] == 1
    assert cache.stats["cache_skip_hits"] == 2
    assert cache.stats["cache_misses"] == 1
    assert cache.stats["cache_evictions"] >= 2

    cache.clear()
    assert cache.size == 0
    assert cache.skip_size == 0


def test_router_result_helpers_and_summary() -> None:
    pure = RouterCompressionResult(
        compressed="small",
        original="very large",
        strategy_used=CompressionStrategy.TEXT,
        routing_log=[
            RoutingDecision(
                content_type=ContentType.PLAIN_TEXT,
                strategy=CompressionStrategy.TEXT,
                original_tokens=10,
                compressed_tokens=4,
            )
        ],
    )
    assert pure.total_original_tokens == 10
    assert pure.total_compressed_tokens == 4
    assert pure.compression_ratio == 0.4
    assert pure.tokens_saved == 6
    assert pure.savings_percentage == 60.0
    assert pure.summary() == "Pure text: 10→4 tokens (60% saved)"

    mixed = RouterCompressionResult(
        compressed="joined",
        original="original",
        strategy_used=CompressionStrategy.MIXED,
        sections_processed=2,
        routing_log=[
            RoutingDecision(
                content_type=ContentType.PLAIN_TEXT,
                strategy=CompressionStrategy.TEXT,
                original_tokens=0,
                compressed_tokens=0,
            ),
            RoutingDecision(
                content_type=ContentType.SEARCH_RESULTS,
                strategy=CompressionStrategy.SEARCH,
                original_tokens=8,
                compressed_tokens=2,
            ),
        ],
    )
    assert mixed.routing_log[0].compression_ratio == 1.0
    assert mixed.summary().startswith("Mixed content: 2 sections, routed to ")


def test_content_signature_and_detection_helpers(monkeypatch: pytest.MonkeyPatch) -> None:
    """Stage-3d (PR5) wired `_detect_content` through the Rust chain
    (`headroom._core.detect_content_type` → magika → unidiff →
    PlainText). The pre-PR5 Python-side `_get_magika_detector`
    fallback path is gone.

    This test asserts the new contract:
    1. The detection helper delegates to the Rust binding.
    2. Whatever `ContentType` the Rust side returns flows back as a
       Python `DetectionResult` with that same `content_type`.
    """
    signature = _create_content_signature("search", "file.py:10:match", language="python")
    assert signature is not None
    assert len(signature.structure_hash) == 24

    # Monkeypatch the Rust binding to return a deterministic fake
    # result; verify _detect_content propagates the content_type
    # tag back as the Python ContentType enum.
    import headroom._core as _core

    # Pin the Rust backend so this test exercises the native delegation
    # path on every platform (Windows now defaults to the pure-Python
    # detector — see content_router._resolve_detect_backend).
    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")

    fake_rust_result = SimpleNamespace(
        content_type="source_code",
        confidence=1.0,
        metadata={},
    )
    monkeypatch.setattr(_core, "detect_content_type", lambda content: fake_rust_result)

    result = _detect_content("def main(): pass")
    assert result.content_type is ContentType.SOURCE_CODE
    assert result.confidence == 1.0
    assert result.metadata == {}


def test_native_detection_remains_bounded_after_success(monkeypatch: pytest.MonkeyPatch) -> None:
    """A successful native call must not disable the watchdog for later calls."""
    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
    monkeypatch.setattr(content_router_module, "_detect_timeout_secs", lambda: 0.01)
    calls = 0

    def _succeeds_then_hangs(_content: str) -> SimpleNamespace:
        nonlocal calls
        calls += 1
        if calls == 1:
            return SimpleNamespace(content_type="plain_text")
        threading.Event().wait()
        raise AssertionError("unreachable")

    monkeypatch.setattr(_core, "detect_content_type", _succeeds_then_hangs)

    assert _detect_content("first").content_type is ContentType.PLAIN_TEXT
    assert _detect_content("second").content_type is ContentType.PLAIN_TEXT
    assert content_router_module._detect_native_unhealthy is True


def test_mixed_content_section_splitting_and_json_extraction() -> None:
    content = "\n".join(
        [
            "Intro paragraph with Several words included for prose detection.",
            "Another line with enough words to read as normal prose today.",
            "Third line adds more prose so the detector sees real text content.",
            "Fourth sentence keeps the count moving higher for prose patterns.",
            "Fifth sentence does the same for mixed content identification.",
            "Sixth sentence seals the prose threshold for the helper.",
            "```python",
            "def main():",
            "    return 1",
            "```",
            '[{"id": 1}]',
            "src/app.py:10:def main():",
            "src/app.py:11:return 1",
        ]
    )
    assert is_mixed_content(content) is True

    sections = split_into_sections(content)
    assert [section.content_type for section in sections] == [
        ContentType.PLAIN_TEXT,
        ContentType.SOURCE_CODE,
        ContentType.JSON_ARRAY,
        ContentType.SEARCH_RESULTS,
    ]
    assert sections[1].language == "python"
    assert sections[1].is_code_fence is True
    assert sections[2].content == '[{"id": 1}]'
    assert sections[3].end_line == 12

    json_block, end_idx = _extract_json_block(["[", '{"id": 1}', "]"], 0)
    assert json_block == '[\n{"id": 1}\n]'
    assert end_idx == 2
    assert _extract_json_block(["{", '"a": 1'], 0) == (None, 0)


def test_short_instruction_with_embedded_json_compresses_without_kompress() -> None:
    rows = [
        {
            "id": i,
            "source": "opencode",
            "status": "ok",
            "message": "routine event with repeated fields",
            "needle": "CRITICAL_NEEDLE_42" if i == 17 else "ordinary",
        }
        for i in range(40)
    ]
    content = "\n".join(
        [
            "Compress this OpenCode context. Preserve critical values.",
            json.dumps(rows, indent=2),
            "Return a concise answer.",
        ]
    )
    router = ContentRouter(
        ContentRouterConfig(
            enable_kompress=False,
            skip_user_messages=False,
        )
    )

    assert is_mixed_content(content) is True
    assert router._determine_strategy(content) is CompressionStrategy.MIXED

    result = router.compress(content, context="CRITICAL_NEEDLE_42 opencode")

    assert result.strategy_used is CompressionStrategy.MIXED
    assert result.tokens_saved > 0
    assert "CRITICAL_NEEDLE_42" in result.compressed
    assert any(
        decision.strategy is CompressionStrategy.SMART_CRUSHER for decision in result.routing_log
    )


def test_extract_json_block_ignores_brackets_inside_strings() -> None:
    """Brackets/braces inside JSON string values must not end the block early.

    Regression: counting raw ``[``/``]``/``{``/``}`` per line treated the
    ``]`` inside ``{"path": "a]b"}`` as a closing bracket, so the array was
    truncated mid-way and the remaining rows leaked into later sections.
    """
    import json as _json

    lines = [
        "[",
        '  {"path": "a]b"},',
        '  {"path": "c"}',
        "]",
    ]
    block, end_idx = _extract_json_block(lines, 0)
    assert end_idx == 3
    assert block is not None
    parsed = _json.loads(block)
    assert parsed == [{"path": "a]b"}, {"path": "c"}]

    # Braces inside a string value must likewise be ignored.
    obj_lines = [
        "{",
        '  "msg": "use {curly} and [square]",',
        '  "n": 1',
        "}",
    ]
    obj_block, obj_end = _extract_json_block(obj_lines, 0)
    assert obj_end == 3
    assert obj_block is not None
    assert _json.loads(obj_block) == {"msg": "use {curly} and [square]", "n": 1}


def test_split_into_sections_keeps_json_array_with_bracket_in_string() -> None:
    """A JSON array embedded in prose stays one JSON section, not fragments.

    With the bracket-in-string bug, the array below split into a truncated
    JSON section plus a stray ``]`` glued onto the trailing prose.
    """
    import json as _json

    content = "\n".join(
        [
            "prose line here that is long enough to matter",
            "[",
            '  {"path": "a]b"},',
            '  {"path": "c"}',
            "]",
            "trailing prose",
        ]
    )

    sections = split_into_sections(content)
    json_sections = [s for s in sections if s.content_type == ContentType.JSON_ARRAY]
    assert len(json_sections) == 1
    parsed = _json.loads(json_sections[0].content)
    assert parsed == [{"path": "a]b"}, {"path": "c"}]


def test_content_router_strategy_and_compress_paths(monkeypatch: pytest.MonkeyPatch) -> None:
    router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 1.0, {}),
    )
    assert router._determine_strategy("code") is CompressionStrategy.PASSTHROUGH
    assert (
        router._strategy_from_detection(DetectionResult(ContentType.SEARCH_RESULTS, 1.0, {}))
        is CompressionStrategy.SEARCH
    )
    assert router._strategy_from_detection_type(ContentType.GIT_DIFF) is CompressionStrategy.DIFF
    assert (
        router._content_type_from_strategy(CompressionStrategy.PASSTHROUGH)
        is ContentType.PLAIN_TEXT
    )

    mixed_result = RouterCompressionResult(
        compressed="mixed",
        original="mixed",
        strategy_used=CompressionStrategy.MIXED,
    )
    pure_result = RouterCompressionResult(
        compressed="pure",
        original="pure",
        strategy_used=CompressionStrategy.TEXT,
    )
    monkeypatch.setattr(router, "_compress_mixed", lambda *args, **kwargs: mixed_result)
    monkeypatch.setattr(router, "_compress_pure", lambda *args, **kwargs: pure_result)

    monkeypatch.setattr(
        router, "_determine_strategy", lambda content, **_kwargs: CompressionStrategy.MIXED
    )
    assert router.compress("mixed") is mixed_result

    monkeypatch.setattr(
        router, "_determine_strategy", lambda content, **_kwargs: CompressionStrategy.TEXT
    )
    assert router.compress("pure") is pure_result
    assert router.compress("   ").strategy_used is CompressionStrategy.PASSTHROUGH


def test_force_kompress_bypasses_content_detection(monkeypatch: pytest.MonkeyPatch) -> None:
    router = ContentRouter()
    router._runtime_force_kompress = True
    monkeypatch.setattr(router, "_force_kompress_ready", lambda: True)
    pure_result = RouterCompressionResult(
        compressed="pure",
        original="pure",
        strategy_used=CompressionStrategy.KOMPRESS,
    )

    monkeypatch.setattr(
        content_router_module,
        "is_mixed_content",
        lambda content: (_ for _ in ()).throw(AssertionError("mixed detection called")),
    )
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: (_ for _ in ()).throw(AssertionError("content detection called")),
    )
    monkeypatch.setattr(router, "_determine_strategy", lambda content: CompressionStrategy.MIXED)
    monkeypatch.setattr(router, "_compress_pure", lambda *args, **kwargs: pure_result)

    assert router.compress("large tool output") is pure_result


def test_normal_compress_path_still_uses_content_detection(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = ContentRouter()
    calls = {"mixed": 0, "detect": 0}
    pure_result = RouterCompressionResult(
        compressed="pure",
        original="pure",
        strategy_used=CompressionStrategy.TEXT,
    )

    def _fake_mixed(content: str) -> bool:
        calls["mixed"] += 1
        return False

    def _fake_detect(content: str) -> DetectionResult:
        calls["detect"] += 1
        return DetectionResult(ContentType.PLAIN_TEXT, 1.0, {})

    monkeypatch.setattr(content_router_module, "is_mixed_content", _fake_mixed)
    monkeypatch.setattr(content_router_module, "_detect_content", _fake_detect)
    monkeypatch.setattr(router, "_compress_pure", lambda *args, **kwargs: pure_result)

    assert router.compress("plain text") is pure_result
    assert calls["mixed"] > 0
    assert calls["detect"] > 0


def test_force_kompress_apply_uses_lightweight_detection(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FakeTokenizer:
        def count_text(self, text: str) -> int:
            return len(text.split())

    router = ContentRouter(ContentRouterConfig(protect_recent_code=2))
    content = " ".join(["plain text payload"] * 80)

    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: (_ for _ in ()).throw(AssertionError("content detection called")),
    )
    monkeypatch.setattr(
        content_router_module,
        "_regex_detect_content_type",
        lambda content: DetectionResult(ContentType.PLAIN_TEXT, 1.0, {}),
    )
    monkeypatch.setattr(
        router,
        "compress",
        lambda content, context="", bias=1.0, precomputed_detection=None: RouterCompressionResult(
            # CCR marker -> the original was stored and is retrievable, so the
            # #1307 reversibility gate accepts this lossy KOMPRESS tool result.
            compressed="compressed <<ccr:tool>>",
            original=content,
            strategy_used=CompressionStrategy.KOMPRESS,
            routing_log=[
                RoutingDecision(
                    content_type=ContentType.PLAIN_TEXT,
                    strategy=CompressionStrategy.KOMPRESS,
                    original_tokens=len(content.split()),
                    compressed_tokens=1,
                )
            ],
        ),
    )

    result = router.apply(
        [{"role": "tool", "content": content}],
        FakeTokenizer(),
        force_kompress=True,
        min_tokens_to_compress=10,
        protect_recent=2,
    )

    assert result.messages[0]["content"] == "compressed <<ccr:tool>>"


def test_force_kompress_apply_lightweight_detection_protects_recent_code(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    class FakeTokenizer:
        def count_text(self, text: str) -> int:
            return len(text.split())

    router = ContentRouter(ContentRouterConfig(protect_recent_code=2))
    content = "\n".join(
        [
            "def generated_function(value):",
            "    if value:",
            "        return str(value)",
        ]
        * 40
    )

    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: (_ for _ in ()).throw(AssertionError("content detection called")),
    )
    monkeypatch.setattr(
        router,
        "compress",
        lambda *args, **kwargs: (_ for _ in ()).throw(
            AssertionError("recent code should be protected")
        ),
    )

    result = router.apply(
        [{"role": "tool", "content": content}],
        FakeTokenizer(),
        force_kompress=True,
        min_tokens_to_compress=10,
        protect_recent=2,
    )

    assert result.messages[0]["content"] == content
    assert result.transforms_applied == ["router:protected:recent_code"]


def test_diff_strategy_does_not_fallback_to_kompress_when_diff_is_noop(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = ContentRouter()
    diff = "diff --git a/a.txt b/a.txt\n--- a/a.txt\n+++ b/a.txt\n@@ -1 +1 @@\n-a\n+a"

    class NoopDiffCompressor:
        def compress(self, content: str, context: str = "") -> SimpleNamespace:
            return SimpleNamespace(compressed=content)

    monkeypatch.setattr(router, "_get_diff_compressor", lambda: NoopDiffCompressor())

    def fail_kompress(*_args: object, **_kwargs: object) -> tuple[str, int]:
        raise AssertionError("Diff compression must not fallback to Kompress")

    monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)

    compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
        diff,
        CompressionStrategy.DIFF,
        context="",
    )

    assert compressed == diff
    assert compressed_tokens == _estimate_tokens(diff)
    assert strategy_chain == ["diff"]


def test_log_strategy_does_not_fallback_to_kompress_when_log_is_noop(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = ContentRouter()
    log = "ERROR one\nERROR two\nERROR three"

    class NoopLogCompressor:
        def compress(self, content: str, bias: float = 1.0) -> SimpleNamespace:
            return SimpleNamespace(compressed=content)

    monkeypatch.setattr(router, "_get_log_compressor", lambda: NoopLogCompressor())

    def fail_kompress(*_args: object, **_kwargs: object) -> tuple[str, int]:
        raise AssertionError("Log compression must not fallback to Kompress")

    monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)

    compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
        log,
        CompressionStrategy.LOG,
        context="",
    )

    assert compressed == log
    assert compressed_tokens == _estimate_tokens(log)
    assert strategy_chain == ["log"]


def test_smart_crusher_log_fallback_skipped_for_invalid_json(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Truncated/invalid JSON must not reach the Log fallback (#1306).

    The native magika detector tags content by shape, so a truncated JSON
    tool output is classified as ``json_array`` and routed to SMART_CRUSHER.
    SmartCrusher returns it unchanged (can't parse broken JSON); Kompress
    passes it through. Without the JSON-validity guard, the LogCompressor
    would then collapse the whole thing to a single CCR-retrieval marker —
    99.9% data loss when CCR retrieval isn't configured. The guard skips
    the Log fallback for invalid JSON so the content passes through verbatim.
    """
    router = ContentRouter(ContentRouterConfig())
    # Truncated JSON object: starts valid, cut mid-string so json.loads fails.
    truncated = '{"rows": [{"address": "Addr1", "name": "PoolA"}, {"address": "Addr2", "nam'

    # SmartCrusher returns no savings (can't parse) → Kompress is a no-op
    # (model not loaded in tests) → Log fallback must be SKIPPED.
    class CollapsingLogCompressor:
        def compress(self, content: str, bias: float = 1.0) -> SimpleNamespace:
            # Simulate the collapse: LogCompressor treats the broken JSON as
            # a multi-line "log" and reduces it to a retrieval marker.
            return SimpleNamespace(
                compressed="\n[2 lines compressed to 0. Retrieve more: hash=deadbeef]"
            )

    monkeypatch.setattr(router, "_get_log_compressor", lambda: CollapsingLogCompressor())
    # Kompress not ready → returns content unchanged (real behavior in tests).
    monkeypatch.setattr(
        router,
        "_try_ml_compressor",
        lambda content, context, question=None: (content, len(content.split())),
    )

    compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
        truncated,
        CompressionStrategy.SMART_CRUSHER,
        context="",
    )

    # Content preserved verbatim — no collapse.
    assert compressed == truncated
    assert CompressionStrategy.LOG.value not in strategy_chain
    assert CompressionStrategy.KOMPRESS.value in strategy_chain


def test_smart_crusher_fallback_skips_kompress_for_valid_json_object(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Kompress must not delete records from parseable single-line JSON (#3673).

    SmartCrusher passes a compact JSON object through when it has no applicable
    array transform. The no-savings fallback used to hand the whole document to
    Kompress; when Kompress dropped a span around ``},{``, the result remained
    valid JSON but silently lost a record.
    """
    router = ContentRouter(ContentRouterConfig())
    payload = json.dumps(
        {
            "domains": [
                {"name": "first", "description": "First collection description"},
                {"name": "second", "description": "Second collection description"},
                {"name": "third", "description": "Third collection description"},
            ]
        },
        separators=(",", ":"),
    )
    kompress_calls: list[str] = []

    class NoopSmartCrusher:
        def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
            return SimpleNamespace(compressed=content)

    class RecordEatingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_calls.append("kompress")
            return SimpleNamespace(
                compressed='{"domains":["record deleted"]}',
                compressed_tokens=1,
            )

    monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())
    monkeypatch.setattr(router, "_get_log_compressor", lambda: None)
    monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())

    result = router.compress(payload)

    assert result.compressed == payload
    assert kompress_calls == []


def test_force_kompress_skips_valid_json_object(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The forced-Kompress fast path keeps the same parseable-JSON guard."""
    router = ContentRouter(ContentRouterConfig(force_kompress_all=True))
    router._runtime_force_kompress = True
    payload = json.dumps({"items": [{"id": 1}, {"id": 2}]}, separators=(",", ":"))
    calls: list[str] = []

    class RecordEatingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            calls.append("kompress")
            return SimpleNamespace(compressed='{"items":[{"id":1}]}', compressed_tokens=1)

    monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())

    result = router.compress(payload)

    assert calls == []
    assert result.compressed == payload


def test_prefixed_same_line_json_skips_kompress(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Embedded JSON must be guarded even when the block itself is not JSON.

    The mixed splitter gives a JSON span its own section only when a line starts
    with ``{``/``[``. With ``Tool result:`` on the same line, embedded routing
    finds the span but SmartCrusher may produce no savings; the full block then
    reaches the Kompress fallback. That final input still contains JSON and must
    not reach the prose model (#3673).
    """
    router = ContentRouter(ContentRouterConfig())
    payload = 'Tool result: {"items":[{"id":1},{"id":2},{"id":3}]}'
    kompress_calls: list[str] = []

    class NoopSmartCrusher:
        def crush(self, content: str, query: str = "", bias: float = 1.0) -> SimpleNamespace:
            return SimpleNamespace(compressed=content)

    class RecordEatingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_calls.append("kompress")
            return SimpleNamespace(
                compressed='Tool result: {"items":[{"id":1}]}',
                compressed_tokens=1,
            )

    monkeypatch.setattr(router, "_get_smart_crusher", lambda: NoopSmartCrusher())
    monkeypatch.setattr(router, "_get_log_compressor", lambda: None)
    monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())

    result = router.compress(payload)

    assert kompress_calls == []
    assert result.compressed == payload


def test_record_stream_of_separate_objects_still_compresses(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Separate top-level objects (search hits, JSONL) are NOT protected.

    Welding two of them yields an unparseable record rather than a silently
    shorter document, so the #3673 invisible-loss argument does not apply.
    Protecting them turns forced Kompress into a no-op on real search payloads
    — ``test_force_kompress_routes_anthropic_tool_result_to_targeted_kompress``
    in tests/test_transforms/test_content_router.py depends on this line.
    """
    router = ContentRouter(ContentRouterConfig())
    stream = " ".join(
        json.dumps({"file": f"src/mod_{i}.py", "line": i, "text": "repeated search payload"})
        for i in range(160)
    )
    kompress_inputs: list[str] = []

    class HalvingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_inputs.append(content)
            out = " ".join(content.split()[::2])
            return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))

    monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())

    out, _tokens = router._try_ml_compressor(stream, context="")

    assert kompress_inputs == [stream]
    assert out != stream


def test_record_array_is_protected_even_beside_a_lone_object(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The protected shape is the record array, at any offset in the block."""
    router = ContentRouter(ContentRouterConfig())
    document = json.dumps({"domains": [{"name": "a"}, {"name": "b"}, {"name": "c"}]})
    block = (
        "The listing call returned two payloads on this line: "
        + json.dumps({"file": "src/mod.py", "line": 1})
        + " and then "
        + document
        + " done."
    )
    kompress_inputs: list[str] = []

    class RecordEatingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_inputs.append(content)
            return SimpleNamespace(compressed="eaten", compressed_tokens=1)

    monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())

    out, _tokens = router._try_ml_compressor(block, context="")

    assert out == block
    assert kompress_inputs == []


def test_protection_scan_stays_linear_on_brace_heavy_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Source and logs are full of ``{`` that never close. The protection guard
    scans at every Kompress boundary; it must not re-read the rest of the block
    for each brace, or large tool output spends its compression time in scans."""
    import headroom.transforms.recursive_json as rj

    router = ContentRouter(ContentRouterConfig())
    line = "    if (flags & MASK) { log.debug(state); retry(ctx, {timeout: 30, max attempts\n"
    block = line * 800  # ~65 KB, two unmatched `{` per line, no JSON at all
    walked = 0
    real = rj._scan_from

    def counting(text, start, known):  # noqa: ANN001, ANN202
        nonlocal walked
        end, n = real(text, start, known)
        walked += n
        return end, n

    monkeypatch.setattr(rj, "_scan_from", counting)
    kompress_inputs: list[str] = []

    class HalvingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_inputs.append(content)
            out = " ".join(content.split()[::2])
            return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))

    monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())

    out, _tokens = router._try_ml_compressor(block, context="")

    # No JSON to protect, so prose compression still runs...
    assert kompress_inputs
    assert out != block
    # ...and every scan of the call together stays a small multiple of one pass
    # (the quadratic walk read hundreds of times the block here).
    assert walked <= 8 * len(block), (walked, len(block))


def test_unfinished_protection_scan_protects_the_whole_block(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """If the scan ever stops at its budget, a record array past that point
    cannot be ruled out, so the block is kept away from the prose model."""
    import headroom.transforms.recursive_json as rj

    monkeypatch.setattr(rj, "_SCAN_BUDGET_PER_CHAR", 0)
    monkeypatch.setattr(rj, "_SCAN_BUDGET_FLOOR", 0)
    router = ContentRouter(ContentRouterConfig())
    block = "Results below. " + json.dumps([{"id": i, "v": "x" * 20} for i in range(5)])
    kompress_inputs: list[str] = []

    class RecordEatingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_inputs.append(content)
            return SimpleNamespace(compressed="eaten", compressed_tokens=1)

    monkeypatch.setattr(router, "_get_kompress", lambda: RecordEatingKompress())

    out, _tokens = router._try_ml_compressor(block, context="")

    assert out == block
    assert kompress_inputs == []


def test_relevance_split_still_runs_for_line_contained_json(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """JSONL keeps each object inside one line, so no window cuts a document and
    the split — which exists for log payloads — proceeds as before."""
    router = ContentRouter(ContentRouterConfig(relevance_split=True))
    payload = "\n".join(
        json.dumps({"level": "info", "msg": f"event {i} with a prose-like description"})
        for i in range(40)
    )
    planned: list[str] = []

    router._relevance_scorer = object()
    router._relevance_scorer_tried = True
    monkeypatch.setattr(
        content_router_module,
        "plan_relevance_split",
        lambda content, query, scorer, **_kw: planned.append(content) or [(True, content)],
    )

    router._relevance_split_compress(payload, "log", "event 3")

    assert planned == [payload]


def test_relevance_split_runs_when_a_record_array_sits_inside_one_window(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Only a STRADDLING span blocks the split.

    A log whose paragraphs happen to embed a small record array is windowed at
    the blank lines, so the array stays whole inside one window. That window is
    safe to score and drop: if it is dropped, the Kompress boundary sees the
    intact array and passes it through. Declining here instead would disable
    the relevance split for most log payloads.
    """
    router = ContentRouter(ContentRouterConfig(relevance_split=True))
    array = json.dumps([{"id": 1, "state": "ok"}, {"id": 2, "state": "ok"}])
    paragraphs = [
        "worker started and connected to the queue, waiting for the first batch",
        "batch 1 finished; per-item results were reported as " + array,
        "worker idle for 30 seconds, then shut down cleanly on SIGTERM",
    ]
    payload = (chr(10) + chr(10)).join(paragraphs)
    planned: list[str] = []
    kompress_inputs: list[str] = []

    class HalvingKompress:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            kompress_inputs.append(content)
            out = " ".join(content.split()[::2])
            return SimpleNamespace(compressed=out, compressed_tokens=len(out.split()))

    router._relevance_scorer = object()
    router._relevance_scorer_tried = True
    monkeypatch.setattr(router, "_get_kompress", lambda: HalvingKompress())
    monkeypatch.setattr(
        content_router_module,
        "plan_relevance_split",
        lambda content, query, scorer, **_kw: (
            planned.append(content)
            or [(False, piece) for piece in content_router_module.segment(content)]
        ),
    )

    router._relevance_split_compress(payload, "log", "worker shutdown")

    # The split ran...
    assert planned == [payload]
    # ...and the window carrying the array reached the Kompress boundary, which
    # passed it through untouched rather than letting the prose model see it.
    carrying = [text for text in kompress_inputs if array in text]
    assert carrying == [], f"a record array reached the prose model: {carrying}"


def test_relevance_split_does_not_fragment_json_payloads(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Relevance windowing must not cut JSON before the Kompress guard.

    A detector can misclassify JSON as LOG/SEARCH. Windowing such a payload at
    blank-line-free boundaries produces fragments that no longer parse, so a
    whole-block JSON check cannot help. Skip the split and let the structural
    JSON boundary preserve the document (#3673).
    """
    router = ContentRouter(ContentRouterConfig(relevance_split=True))
    payload = json.dumps(
        {
            "items": [
                {"id": index, "value": f"record {index} with prose-like description"}
                for index in range(8)
            ]
        },
        indent=2,
    )
    kompress_inputs: list[str] = []

    class DeterministicScorer:
        def score_batch(self, items: list[str], _context: str) -> list[SimpleNamespace]:
            return [SimpleNamespace(score=0.0) for _ in items]

    def record_kompress(
        content: str,
        _context: str,
        question: str | None = None,
    ) -> tuple[str, int]:
        kompress_inputs.append(content)
        return content, _estimate_tokens(content)

    router._relevance_scorer = DeterministicScorer()
    router._relevance_scorer_tried = True
    monkeypatch.setattr(router, "_try_ml_compressor", record_kompress)

    assert router._relevance_split_compress(payload, "log", "find record 3") is None
    assert kompress_inputs == []


# ---------------------------------------------------------------------------
# MIXED false-positive on source code: Python with dict literals + docstrings
# triggers ``has_json_blocks`` and ``has_prose`` in ``is_mixed_content``,
# misclassifying pure code as MIXED.  The native detector (magika) correctly
# identifies it as SOURCE_CODE.  ``_determine_strategy`` must trust the
# detector over the regex heuristics when confidence is high.
# ---------------------------------------------------------------------------


def test_mixed_false_positive_on_source_code_overridden_by_detector(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Python code with dicts + docstrings must not be routed to MIXED.

    ``is_mixed_content`` returns True (dict literals → has_json_blocks,
    docstrings → has_prose), but the native detector says SOURCE_CODE
    with high confidence.  The detector must win.
    """
    router = ContentRouter(ContentRouterConfig())

    # Simulate the false positive: is_mixed_content says True, detector
    # says SOURCE_CODE with confidence 1.0 (real magika behaviour).
    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: True)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 1.0, {}),
    )

    strategy = router._determine_strategy("python code with dicts")
    # With prefer_code_aware_for_code=True (default), source code routes to CodeAware.
    assert strategy is CompressionStrategy.CODE_AWARE
    assert strategy is not CompressionStrategy.MIXED


def test_mixed_still_used_when_detector_says_plain_text(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Genuine mixed content (detector says PLAIN_TEXT) still uses MIXED.

    The override only fires when the detector says SOURCE_CODE.  For
    actual mixed content (prose + JSON + code fences), the detector
    returns PLAIN_TEXT or another non-code type, and MIXED is correct.
    """
    router = ContentRouter(ContentRouterConfig())

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: True)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.PLAIN_TEXT, 0.9, {}),
    )

    strategy = router._determine_strategy("genuinely mixed content")
    assert strategy is CompressionStrategy.MIXED


def test_mixed_still_used_when_detector_confidence_below_threshold(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Low-confidence SOURCE_CODE detection does not override MIXED.

    If the detector is uncertain (confidence < 0.8), we keep the MIXED
    path rather than risking a false override.
    """
    router = ContentRouter(ContentRouterConfig())

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: True)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 0.5, {}),
    )

    strategy = router._determine_strategy("uncertain code")
    assert strategy is CompressionStrategy.MIXED


def test_source_code_passthrough_preserves_content_verbatim(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Code detected as SOURCE_CODE with CodeAware disabled → passthrough.

    When ``prefer_code_aware_for_code=False`` (default), code must pass
    through unmangled.  Previously it fell to KOMPRESS which can destroy
    code semantics (98% compression, 11% recall on large blobs).  Now
    it explicitly uses PASSTHROUGH.
    """
    router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 1.0, {}),
    )

    code = "def hello():\n    print('world')\n"
    result = router.compress(code)
    assert result.compressed == code
    assert result.strategy_used is CompressionStrategy.PASSTHROUGH


def test_source_code_code_aware_enabled_uses_code_aware(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """When CodeAware is explicitly enabled, it is used for source code.

    The PASSTHROUGH fallback only applies when ``prefer_code_aware_for_code``
    is False.  If a user explicitly enables it, CODE_AWARE is used.
    """
    router = ContentRouter(
        ContentRouterConfig(
            enable_code_aware=True,
            prefer_code_aware_for_code=True,
        )
    )

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 1.0, {}),
    )

    strategy = router._determine_strategy("def foo(): pass")
    assert strategy is CompressionStrategy.CODE_AWARE


# ---------------------------------------------------------------------------
# MIXED false-positive on a complete HTML document: embedded JSON or ordinary
# body prose triggers ``has_json_blocks``/``has_prose`` in ``is_mixed_content``,
# misclassifying a page with a clear DOCTYPE/``<html>`` root as MIXED.  The
# detector says HTML.  ``_determine_strategy`` must trust it over the regex
# heuristics, same as the SOURCE_CODE case above. See #3608, #3609.
# ---------------------------------------------------------------------------

# Product page with inline JSON-LD: the shape from #3608. The ``ld+json`` block
# trips ``has_json_blocks``, the article copy trips ``has_prose``, so
# ``is_mixed_content`` says True even though the document is unmistakably HTML.
# trafilatura keeps the prose and drops the ``<script>`` payload, which is why
# the tool-role guard below has to treat HTML as lossy-unmarked.
MIXED_HTML_PRODUCT_PAGE = """<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<title>Field Jacket | Example Outfitters</title>
<script type="application/ld+json">
{
  "@context": "https://schema.org",
  "@type": "Product",
  "name": "Field Jacket",
  "sku": "SKU-48213",
  "offers": {"@type": "Offer", "price": "189.00", "priceCurrency": "USD"},
  "aggregateRating": {"@type": "AggregateRating", "ratingValue": "4.7", "reviewCount": "132"}
}
</script>
</head>
<body>
<header><nav><a href="/">Home</a> <a href="/jackets">Jackets</a></nav></header>
<article>
<h1>Field Jacket</h1>
<p>The Field Jacket is cut from a waxed cotton shell that sheds light rain without feeling stiff.</p>
<p>Two chest pockets and two hand pockets close with brass snaps and the collar stands up on its own.</p>
<p>We recommend sizing down if you plan to layer it over a sweater during colder months.</p>
<p>Every jacket is finished by hand in our workshop and ships within two business days.</p>
<p>Returns are accepted for thirty days as long as the tags are still attached to the garment.</p>
<p>Care instructions are printed on the inner label so sponge clean only and never machine wash.</p>
</article>
<footer><p>Example Outfitters, 42 Market Street</p></footer>
</body>
</html>"""

# What an HTML extractor keeps from that page: prose only, no JSON-LD facts.
HTML_PROSE_ONLY = "\n\n".join(
    [
        "Field Jacket",
        "The Field Jacket is cut from a waxed cotton shell that sheds light rain without "
        "feeling stiff.",
        "Two chest pockets and two hand pockets close with brass snaps and the collar stands "
        "up on its own.",
        "We recommend sizing down if you plan to layer it over a sweater during colder months.",
        "Every jacket is finished by hand in our workshop and ships within two business days.",
        "Returns are accepted for thirty days as long as the tags are still attached to the "
        "garment.",
        "Care instructions are printed on the inner label so sponge clean only and never "
        "machine wash.",
        "Example Outfitters, 42 Market Street",
    ]
)


# A prose answer that embeds a fenced complete HTML example. Seven sentences
# trip ``has_prose`` and the fence trips ``has_code_fences``, so
# ``is_mixed_content`` says True; the tags inside the fence also make the
# detector report HTML. The override must not treat the whole answer as a page.
FENCED_HTML_EXAMPLE = """Here is how the page should look. Keep the header and the footer.
The layout uses a centered container and a sticky header.
Colours come from the theme variables defined at the top of the stylesheet.
Do not rename the ids, the script queries them directly.
The footer should stay at the bottom on short pages.
Test it in both light and dark mode before shipping it.
If the cards wrap badly, reduce the gap between them.

```html
<!DOCTYPE html>
<html lang="en">
<head><meta charset="utf-8"><title>Demo</title></head>
<body>
<div id="header">Header</div>
<div id="nav">Nav</div>
<div id="main">
  <div class="card">Card one</div>
  <div class="card">Card two</div>
</div>
<div id="footer">Footer</div>
</body>
</html>
```

Save that as index.html and open it in a browser.
The `div` ids are what the JavaScript expects, so do not rename them.
"""


def _stub_html_extractor(extracted: str) -> tuple[SimpleNamespace, list[str]]:
    """Deterministic stand-in for HTMLExtractor (no trafilatura needed)."""
    seen: list[str] = []

    def _extract(html: str) -> SimpleNamespace:
        seen.append(html)
        return SimpleNamespace(extracted=extracted)

    return SimpleNamespace(extract=_extract), seen


def test_mixed_html_product_page_routes_to_html() -> None:
    """A complete HTML page with inline JSON-LD must reach the HTML strategy.

    Both signals come from the real code: ``is_mixed_content`` and the
    pure-Python detector (the same one ``_detect_content`` falls back to on
    Windows). No monkeypatching, so the heuristic shapes that caused #3608 are
    the ones under test.
    """
    from headroom.transforms.content_detector import detect_content_type

    router = ContentRouter(ContentRouterConfig())

    assert is_mixed_content(MIXED_HTML_PRODUCT_PAGE) is True
    detection = detect_content_type(MIXED_HTML_PRODUCT_PAGE)
    assert detection.content_type is ContentType.HTML

    strategy = router._determine_strategy(MIXED_HTML_PRODUCT_PAGE, mixed=True, detection=detection)
    assert strategy is CompressionStrategy.HTML


def test_html_override_has_no_second_confidence_gate() -> None:
    """The router trusts the detector's HTML type without re-checking confidence.

    ``detect_content_type`` only returns HTML when its own score clears 0.7, and
    the native backend hardcodes confidence 1.0, so a threshold here would never
    filter anything — it only read as a real guard. This pins the single gate so
    it isn't re-added as dead code.
    """
    router = ContentRouter(ContentRouterConfig())

    strategy = router._determine_strategy(
        MIXED_HTML_PRODUCT_PAGE,
        mixed=True,
        detection=DetectionResult(ContentType.HTML, 0.5, {}),
    )
    assert strategy is CompressionStrategy.HTML


def test_fenced_html_example_stays_mixed() -> None:
    """A fenced HTML example inside prose must not be extracted as a page.

    The detector reads the tags inside the fence and reports HTML, and the
    answer is mixed, so without the top-level check the whole response goes to
    the HTML extractor and the surrounding instructions and fence lines are
    lost. Both signals come from the real detector and the real heuristic.
    """
    from headroom.transforms.content_detector import detect_content_type

    router = ContentRouter(ContentRouterConfig())

    assert is_mixed_content(FENCED_HTML_EXAMPLE) is True
    detection = detect_content_type(FENCED_HTML_EXAMPLE)
    assert detection.content_type is ContentType.HTML

    strategy = router._determine_strategy(FENCED_HTML_EXAMPLE, mixed=True, detection=detection)
    assert strategy is CompressionStrategy.MIXED


def test_tool_html_with_inline_jsonld_kept_verbatim(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """apply() must not replace tool HTML with prose that lost its JSON-LD (#3627).

    Routing now sends this page to the HTML extractor, which keeps the article
    and drops SKU, price and rating without storing the original or emitting a
    CCR marker. The tool-ground-truth guard has to keep the page verbatim.
    """
    router = ContentRouter(ContentRouterConfig())
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.HTML, 1.0, {}),
    )
    extractor, seen = _stub_html_extractor(HTML_PROSE_ONLY)
    monkeypatch.setattr(router, "_get_html_extractor", lambda: extractor)

    result = router.apply(
        [{"role": "tool", "tool_call_id": "call_bash_1", "content": MIXED_HTML_PRODUCT_PAGE}],
        _ChurnTokenizer(),
        protect_recent=0,
        protect_analysis_context=False,
    )

    assert seen, "extractor never ran: routing did not reach the HTML strategy"
    out = result.messages[0]["content"]
    assert out == MIXED_HTML_PRODUCT_PAGE
    for fact in ("SKU-48213", "189.00", "4.7"):
        assert fact in out


def test_assistant_html_still_compressed(monkeypatch: pytest.MonkeyPatch) -> None:
    """The HTML guard is scoped to tool ground truth, like the other lossy paths."""
    router = ContentRouter(ContentRouterConfig())
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.HTML, 1.0, {}),
    )
    extractor, seen = _stub_html_extractor(HTML_PROSE_ONLY)
    monkeypatch.setattr(router, "_get_html_extractor", lambda: extractor)

    result = router.apply(
        [{"role": "assistant", "content": MIXED_HTML_PRODUCT_PAGE}],
        _ChurnTokenizer(),
        protect_recent=0,
        protect_analysis_context=False,
    )

    assert seen, "extractor never ran: routing did not reach the HTML strategy"
    assert result.messages[0]["content"] == HTML_PROSE_ONLY


def test_html_strategy_without_extractor_passes_through(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Without the ``[html]`` extra (no trafilatura) HTML stays untouched."""
    router = ContentRouter(ContentRouterConfig())
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.HTML, 1.0, {}),
    )
    monkeypatch.setattr(router, "_get_html_extractor", lambda: None)

    result = router.compress(MIXED_HTML_PRODUCT_PAGE)
    assert result.compressed == MIXED_HTML_PRODUCT_PAGE


def test_source_code_passthrough_does_not_invoke_kompress(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """PASSTHROUGH for code must never call _try_ml_compressor.

    KOMPRESS on large code blobs can destroy semantics.  The PASSTHROUGH
    path must be a pure passthrough — no ML compressor invocation.
    """
    router = ContentRouter(ContentRouterConfig(prefer_code_aware_for_code=False))

    monkeypatch.setattr(content_router_module, "is_mixed_content", lambda content: False)
    monkeypatch.setattr(
        content_router_module,
        "_detect_content",
        lambda content: DetectionResult(ContentType.SOURCE_CODE, 1.0, {}),
    )

    def fail_kompress(*_args: object, **_kwargs: object) -> tuple[str, int]:
        raise AssertionError("PASSTHROUGH must not invoke Kompress")

    monkeypatch.setattr(router, "_try_ml_compressor", fail_kompress)

    code = "def hello():\n    return 42\n"
    result = router.compress(code)
    assert result.compressed == code
    assert result.strategy_used is CompressionStrategy.PASSTHROUGH


# ---------------------------------------------------------------------------
# Cache-safety tests for _process_content_blocks. These pin down the
# block-level invariants that protect upstream prefix caches:
#
#   * cache_control on a block is the client's explicit cache breakpoint —
#     never modified, regardless of role/type.
#   * assistant text blocks are part of the cache prefix in subsequent
#     turns; default-skipped, opt-in via compress_assistant_text_blocks.
#   * user/system text blocks are the prompt; never modified.
#   * tool/function text blocks are tool outputs; freely compressed.
#   * min_chars threshold gates short blocks.
# ---------------------------------------------------------------------------


def _make_router_with_mock_compress(monkeypatch: pytest.MonkeyPatch) -> ContentRouter:
    """Return a ContentRouter whose compress() always emits a half-length
    ``[compressed]`` payload at ratio 0.5 (passes the < min_ratio check)."""
    router = ContentRouter(ContentRouterConfig())

    def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
        return SimpleNamespace(
            compressed=content[: len(content) // 2] + "[compressed]",
            compression_ratio=0.5,
            strategy_used=SimpleNamespace(value="text"),
        )

    monkeypatch.setattr(router, "compress", fake_compress)
    return router


def test_text_block_cache_control_protected_with_assistant_optin(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "A" * 1000
    msg = {
        "role": "assistant",
        "content": [
            {"type": "text", "text": long_text, "cache_control": {"type": "ephemeral"}},
            {"type": "text", "text": "B" * 1000},
        ],
    }
    counts: dict[str, int] = {
        "excluded_tool": 0,
        "user_msg": 0,
        "small": 0,
        "recent_code": 0,
        "analysis_ctx": 0,
        "ratio_too_high": 0,
        "non_string": 0,
        "content_blocks": 0,
    }
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        route_counts=counts,
        compress_assistant_text_blocks=True,
    )
    blocks = result["content"]
    # cache_control'd block: untouched (defense in depth)
    assert blocks[0] == msg["content"][0]
    assert blocks[0]["text"] == long_text
    # Sibling non-cache_control'd block: compressed under opt-in
    assert "[compressed]" in blocks[1]["text"]
    assert counts["cache_control_protected"] == 1


def test_tool_result_cache_control_protected(monkeypatch: pytest.MonkeyPatch) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "Z" * 1000
    msg = {
        "role": "user",
        "content": [
            {
                "type": "tool_result",
                "tool_use_id": "abc",
                "content": long_text,
                "cache_control": {"type": "ephemeral"},
            }
        ],
    }
    counts: dict[str, int] = {}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        route_counts=counts,
        messages_from_end=3,
    )
    # cache_control hard-skip applies to tool_result too, on any message the
    # provider may already hold under that key (anything but the final one).
    assert result["content"][0]["content"] == long_text
    assert counts["cache_control_protected"] == 1


def test_tool_result_cache_control_compressed_in_final_message(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # The request's final user message has never been forwarded, so its
    # cache_control marker is the client staking out NEXT turn's breakpoint,
    # not a key the provider holds. Claude Code stamps its newest tool_result
    # every turn; protecting it here skipped the freshest tool output on the
    # one turn it was fresh.
    router = ContentRouter(ContentRouterConfig())

    def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
        # A real (hashable) lossless strategy: tool_result compression runs the
        # reversibility gate, which looks the strategy up in a frozenset.
        return SimpleNamespace(
            compressed=content[: len(content) // 2] + "[compressed]",
            compression_ratio=0.5,
            strategy_used=CompressionStrategy.LOG,
        )

    monkeypatch.setattr(router, "compress", fake_compress)
    long_text = "Z" * 1000
    msg = {
        "role": "user",
        "content": [
            {
                "type": "tool_result",
                "tool_use_id": "abc",
                "content": long_text,
                "cache_control": {"type": "ephemeral"},
            }
        ],
    }
    counts: dict[str, int] = {}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        route_counts=counts,
        messages_from_end=1,
        prefix_replay_guaranteed=True,
    )
    out = result["content"][0]
    assert out["content"].endswith("[compressed]")
    # The marker rides on the compressed block: that is what the provider
    # caches, and what the frozen prefix replays next turn.
    assert out["cache_control"] == {"type": "ephemeral"}
    assert "cache_control_protected" not in counts
    # Assistant blocks in the final position keep the hard skip: they are
    # echoed back and were not the tool output this exception is for.
    amsg = {
        "role": "assistant",
        "content": [{"type": "text", "text": long_text, "cache_control": {"type": "ephemeral"}}],
    }
    aresult = router._process_content_blocks(
        amsg,
        amsg["content"],
        "",
        [],
        set(),
        set(),
        messages_from_end=1,
        compress_assistant_text_blocks=True,
        prefix_replay_guaranteed=True,
    )
    assert aresult["content"][0]["text"] == long_text
    # A marked user TEXT block in the final position is the user's prompt,
    # not tool output: it keeps the hard skip even with user compression on.
    umsg = {
        "role": "user",
        "content": [{"type": "text", "text": long_text, "cache_control": {"type": "ephemeral"}}],
    }
    uresult = router._process_content_blocks(
        umsg,
        umsg["content"],
        "",
        [],
        set(),
        set(),
        messages_from_end=1,
        skip_user=False,
        prefix_replay_guaranteed=True,
    )
    assert uresult["content"][0]["text"] == long_text


_FRESH_CC_OUTPUT = "tool output line " * 200


def _fresh_cc_router(monkeypatch: pytest.MonkeyPatch) -> tuple[ContentRouter, list[str]]:
    router = ContentRouter(ContentRouterConfig())
    calls: list[str] = []

    def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
        calls.append(content)
        return SimpleNamespace(
            compressed=content[: len(content) // 2] + "[compressed]",
            compression_ratio=0.5,
            strategy_used=CompressionStrategy.LOG,
        )

    monkeypatch.setattr(router, "compress", fake_compress)
    return router, calls


def _fresh_cc_turn1() -> list[dict]:
    return [
        {"role": "user", "content": "run the tests"},
        {
            "role": "assistant",
            "content": [{"type": "tool_use", "id": "t1", "name": "bash", "input": {}}],
        },
        {
            "role": "user",
            "content": [
                {
                    "type": "tool_result",
                    "tool_use_id": "t1",
                    "content": _FRESH_CC_OUTPUT,
                    "cache_control": {"type": "ephemeral"},
                }
            ],
        },
    ]


def _fresh_cc_turn2() -> list[dict]:
    return _fresh_cc_turn1() + [
        {"role": "assistant", "content": "done"},
        {"role": "user", "content": "now fix the lint"},
    ]


def test_fresh_cache_control_stays_hard_skipped_without_replay_guarantee(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # A caller that hands apply() the client's raw transcript every turn with
    # no frozen-prefix overlay (SDK client, evals, a bare router). Had turn 1
    # compressed the marked block, turn 2 would hard-skip it (no longer final)
    # and forward the original bytes, busting the key the marker names. So
    # without the guarantee the block is never touched, on either turn.
    router, calls = _fresh_cc_router(monkeypatch)
    t1 = router.apply(_fresh_cc_turn1(), _ChurnTokenizer(), model_limit=100_000)
    assert t1.messages[2]["content"][0]["content"] == _FRESH_CC_OUTPUT
    t2 = router.apply(_fresh_cc_turn2(), _ChurnTokenizer(), model_limit=100_000)
    assert t2.messages[2]["content"][0]["content"] == _FRESH_CC_OUTPUT
    assert calls == []


def test_fresh_cache_control_compressed_under_replay_guarantee_is_stable(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    from headroom.cache.prefix_tracker import overlay_cached_prefix

    router, _ = _fresh_cc_router(monkeypatch)
    kw = {"model_limit": 100_000, "prefix_replay_guaranteed": True}
    t1 = router.apply(_fresh_cc_turn1(), _ChurnTokenizer(), **kw)
    forwarded = t1.messages[2]["content"][0]
    assert forwarded["content"].endswith("[compressed]")
    assert forwarded["cache_control"] == {"type": "ephemeral"}
    # Retried final message (the client resends the same transcript, e.g.
    # after a 529): byte-identical, so the key written on the first attempt
    # still matches.
    retry = router.apply(_fresh_cc_turn1(), _ChurnTokenizer(), **kw)
    assert retry.messages[2] == t1.messages[2]
    # Next turn the block is no longer final and contract 1 hard-skips it, so
    # the router alone emits the client's original bytes: this is the reversal
    # the guarantee exists for.
    t2_original = _fresh_cc_turn2()
    t2 = router.apply(t2_original, _ChurnTokenizer(), **kw)
    assert t2.messages[2]["content"][0]["content"] == _FRESH_CC_OUTPUT
    # The guaranteeing caller replays last turn's forwarded prefix over the
    # pipeline output (what the proxy does in finalize_turn), restoring the
    # compressed bytes the provider cached under that marker.
    replayed = overlay_cached_prefix(t2.messages, t2_original, _fresh_cc_turn1(), t1.messages)
    assert replayed[2] == t1.messages[2]
    assert replayed[3:] == t2.messages[3:]


def test_assistant_text_blocks_skipped_by_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "X" * 1000
    msg = {"role": "assistant", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
    )
    # Default OFF: assistant text untouched, restoring pre-#431 cache safety
    assert result["content"][0]["text"] == long_text


def test_assistant_text_blocks_opt_in_compresses(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "Y" * 1000
    msg = {"role": "assistant", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        compress_assistant_text_blocks=True,
    )
    assert "[compressed]" in result["content"][0]["text"]


def test_user_text_blocks_never_compressed_even_with_assistant_optin(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "U" * 1000
    msg = {"role": "user", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        compress_assistant_text_blocks=True,  # MUST NOT bleed into user
    )
    assert result["content"][0]["text"] == long_text


def test_system_text_blocks_skipped_when_skip_system_true(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "S" * 1000
    msg = {"role": "system", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        skip_system=True,
        compress_assistant_text_blocks=True,
    )
    assert result["content"][0]["text"] == long_text


def test_tool_role_text_blocks_compressed_by_default(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = ContentRouter(ContentRouterConfig())
    long_text = "T" * 1000
    recoverable = "T" * 100 + " [compressed] <<ccr:9f3a21>>"
    monkeypatch.setattr(
        router,
        "compress",
        lambda *a, **k: SimpleNamespace(
            compressed=recoverable,
            compression_ratio=0.5,
            strategy_used=CompressionStrategy.TEXT,
        ),
    )
    msg = {"role": "tool", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
    )
    # tool role ≈ tool output — not protected like user/system text; a
    # recoverable result is accepted (an unmarked lossy one is gated, #1307).
    assert result["content"][0]["text"] == recoverable


def test_unknown_role_text_blocks_skipped_for_safety(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    long_text = "Q" * 1000
    msg = {"role": "developer", "content": [{"type": "text", "text": long_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        compress_assistant_text_blocks=True,
    )
    # Unknown role: be safe, don't compress
    assert result["content"][0]["text"] == long_text


def test_min_chars_gates_short_blocks(monkeypatch: pytest.MonkeyPatch) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    short_text = "tiny"
    msg = {"role": "tool", "content": [{"type": "text", "text": short_text}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
        min_chars=500,
    )
    assert result["content"][0]["text"] == short_text


def test_pinning_skips_already_compressed(monkeypatch: pytest.MonkeyPatch) -> None:
    router = _make_router_with_mock_compress(monkeypatch)
    pinned = "Retrieve more: hash=abc " + "x" * 1000
    msg = {"role": "tool", "content": [{"type": "text", "text": pinned}]}
    result = router._process_content_blocks(
        msg,
        msg["content"],
        "",
        [],
        set(),
        set(),
    )
    # Already-compressed marker keeps proxy idempotent across turns
    assert result["content"][0]["text"] == pinned


def test_detect_backend_env_python_forces_python_path(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """HEADROOM_DETECT_BACKEND=python forces the pure-Python regex path."""
    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")

    called = []

    def _record(content: str):  # type: ignore[return]
        called.append(content)
        raise AssertionError("native must not be called with python backend")

    monkeypatch.setattr(_core, "detect_content_type", _record)

    # Should not raise — native detector must be bypassed entirely.
    result = _detect_content('[{"id": 1}]')
    assert result.content_type is ContentType.JSON_ARRAY
    assert called == [], "native detect_content_type was called despite python backend"


def test_detect_backend_env_override(monkeypatch: pytest.MonkeyPatch) -> None:
    """HEADROOM_DETECT_BACKEND pins the detector on any platform."""
    resolve = content_router_module._resolve_detect_backend

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")
    assert resolve() == "python"

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "RUST")  # case-insensitive
    assert resolve() == "rust"

    # Unrecognized values fall back to the platform default.
    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "bogus")
    monkeypatch.setattr(content_router_module.sys, "platform", "linux")
    assert resolve() == "rust"


def test_detect_backend_defaults_to_python_on_windows(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Windows defaults to the pure-Python detector (native ONNX hang, #845)."""
    monkeypatch.delenv("HEADROOM_DETECT_BACKEND", raising=False)

    monkeypatch.setattr(content_router_module.sys, "platform", "win32")
    assert content_router_module._resolve_detect_backend() == "python"

    monkeypatch.setattr(content_router_module.sys, "platform", "linux")
    assert content_router_module._resolve_detect_backend() == "rust"


def test_detect_content_python_backend_skips_native(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The python backend must not touch the native detector at all."""
    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")

    def _boom(_content: str) -> None:
        raise AssertionError("native detector must not be called")

    monkeypatch.setattr(_core, "detect_content_type", _boom)

    result = _detect_content('[{"id": 1}, {"id": 2}]')
    assert result.content_type is ContentType.JSON_ARRAY


# ---------------------------------------------------------------------------
# Cache-churn fix: HEADROOM_FREEZE_BLOCK_DECISION (default off).
#
# Root cause: ``min_ratio`` drifts every turn with context pressure, so a
# block whose compression ratio sits in [aggressive, relaxed) is compressed
# on a low-pressure turn and downgraded to passthrough on a high-pressure
# turn (or vice-versa). The block's prefix bytes flap across turns ⇒
# cache_write churn. The fix freezes the per-block compress/passthrough
# verdict on first sighting against the FIXED aggressive threshold.
# ---------------------------------------------------------------------------


class _ChurnTokenizer:
    """Word-count tokenizer; apply() only calls ``count_text``."""

    def count_text(self, text: str) -> int:
        return len(str(text).split())


def _churn_router(monkeypatch: pytest.MonkeyPatch, ratio: float) -> ContentRouter:
    """Router whose compress() always emits a deterministic payload at a
    fixed ratio. ``min_chars`` thresholds are relaxed so a 200-word tool
    message reaches the compression path."""
    cfg = ContentRouterConfig(min_ratio_relaxed=0.85, min_ratio_aggressive=0.65)
    router = ContentRouter(cfg)

    def fake_compress(content, context: str = "", bias: float = 1.0, precomputed_detection=None):
        return SimpleNamespace(
            # CCR marker keeps the compression recoverable so the #1307
            # tool-reversibility guard preserves it instead of restoring original.
            compressed="[C]" + content[:20] + " <<ccr:t>>",
            compression_ratio=ratio,
            strategy_used=CompressionStrategy.TEXT,
        )

    monkeypatch.setattr(router, "compress", fake_compress)
    # Don't let lazy ML model state interfere with the model_ready signal.
    monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)
    return router


def _run_turn(router: ContentRouter, content: str, tokens_before: int):
    """Drive apply() for a single message with a rising context-pressure
    knob. model_limit fixed at 1000 so larger tokens_before ⇒ higher
    pressure ⇒ tighter min_ratio."""
    messages = [{"role": "tool", "tool_call_id": "t1", "content": content}]
    result = router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=1000,
        # Force a known tokens_before by padding nothing — apply recomputes
        # tokens_before from the messages, so we instead steer pressure via
        # model_limit below. (kept for clarity)
    )
    return result.messages[0]["content"]


def _content_of_n_words(n: int) -> str:
    return " ".join(f"w{i}" for i in range(n))


def test_freeze_off_is_byte_identical_flapping_baseline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(b) Flag OFF (default): a mid-zone block (ratio 0.75) flaps —
    compressed on a low-pressure turn, downgraded to passthrough once
    pressure tightens min_ratio below the ratio. This is the legacy churn
    and must be preserved byte-for-byte when the flag is off."""
    monkeypatch.delenv("HEADROOM_FREEZE_BLOCK_DECISION", raising=False)
    router = _churn_router(monkeypatch, ratio=0.75)
    content = _content_of_n_words(200)

    # Turn 1: low pressure (small model_limit denom via large limit) ->
    # min_ratio ~ relaxed (0.85). 0.75 < 0.85 -> compresses.
    low = router.apply(
        [{"role": "tool", "tool_call_id": "t1", "content": content}],
        _ChurnTokenizer(),
        model_limit=100000,  # pressure ~ 0 -> min_ratio ~ 0.85
    ).messages[0]["content"]
    assert low.startswith("[C]"), "turn1 should compress at low pressure"

    # Turn 2: high pressure -> min_ratio ~ aggressive (0.65). 0.75 < 0.65
    # is False -> cache-hit path downgrades via move_to_skip -> passthrough.
    high = router.apply(
        [{"role": "tool", "tool_call_id": "t1", "content": content}],
        _ChurnTokenizer(),
        model_limit=100,  # 200 words >> 100 -> pressure 1.0 -> min_ratio 0.65
    ).messages[0]["content"]
    assert high == content, "turn2 should flip to passthrough (legacy churn)"
    # The flap: bytes changed across turns.
    assert low != high


def test_freeze_on_pins_compress_verdict_across_turns(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(a) Flag ON: a block that compresses on first sighting keeps the SAME
    verdict AND the SAME bytes on later turns despite rising pressure /
    drifting min_ratio. Use ratio 0.6 (< aggressive 0.65) so it compresses
    under the frozen threshold."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = _churn_router(monkeypatch, ratio=0.60)
    content = _content_of_n_words(200)

    outs = []
    # Rising pressure across turns: model_limit shrinks -> min_ratio drifts
    # from ~0.85 down to ~0.65. Without freeze this is the flap zone.
    for limit in (100000, 1000, 300, 100):
        out = router.apply(
            [{"role": "tool", "tool_call_id": "t1", "content": content}],
            _ChurnTokenizer(),
            model_limit=limit,
        ).messages[0]["content"]
        outs.append(out)

    assert all(o.startswith("[C]") for o in outs), "verdict must stay compress"
    assert len(set(outs)) == 1, "bytes must be identical across all turns"


def test_freeze_on_pins_passthrough_verdict_across_turns(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(a) Flag ON, mid-zone block (ratio 0.75): the RELAXED first-sighting
    threshold (0.85) compresses it on turn 1 (low pressure), freezes the
    compress verdict, and PINS it compressed on every later turn — even as
    rising pressure pulls ``min_ratio`` below 0.75, where the flag-off path
    would ``move_to_skip`` and bust the prefix cache. So the bytes stay
    compressed and identical, and the freeze-pin counter records the
    busts it avoided. (Contrast with the flag-off flapping baseline.)"""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = _churn_router(monkeypatch, ratio=0.75)
    content = _content_of_n_words(200)

    outs = []
    for limit in (100000, 1000, 300, 100):
        out = router.apply(
            [{"role": "tool", "tool_call_id": "t1", "content": content}],
            _ChurnTokenizer(),
            model_limit=limit,
        ).messages[0]["content"]
        outs.append(out)

    assert all(o.startswith("[C]") for o in outs), "verdict must stay compressed"
    assert len(set(outs)) == 1, "bytes identical (always compressed) across turns"
    assert router._freeze_pin_hits > 0, "freeze must pin compress over a tightening min_ratio"


def test_model_not_ready_passthrough_is_not_frozen(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Caveat (1): a block that passes through ONLY because the ML model is
    not ready must NOT have its skip verdict frozen — once the model is
    ready it must be re-evaluated. We simulate not-ready (compress returns
    the content unchanged at ratio 1.0) then ready (real compression)."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    cfg = ContentRouterConfig()
    router = ContentRouter(cfg)
    content = _content_of_n_words(200)

    # Phase 1: model NOT ready. _try_ml_compressor returns content unchanged
    # -> ratio 1.0 (passthrough). Verdict must NOT be frozen as skip.
    monkeypatch.setattr(router, "_kompress_model_ready", lambda: False)
    monkeypatch.setattr(
        router,
        "compress",
        lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
            compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT
        ),
    )
    out1 = router.apply(
        [{"role": "tool", "tool_call_id": "t1", "content": content}],
        _ChurnTokenizer(),
        model_limit=1000,
    ).messages[0]["content"]
    assert out1 == content, "not-ready -> passthrough"
    # The verdict store must NOT contain a frozen skip for this block.
    assert all(v is True for v in router._frozen_verdicts.values()) or (
        not router._frozen_verdicts
    ), "not-ready passthrough must not be frozen as skip"

    # Phase 2: model now ready, real compression below the aggressive
    # threshold. Because the verdict was never frozen, the block must now be
    # re-evaluated and compress (not stuck on a frozen passthrough). Note:
    # the legacy byte skip-cache is keyed on the same content; we clear it to
    # isolate the verdict behaviour (the freeze fix governs the verdict, not
    # the existing TTL byte cache).
    router._cache.clear()
    monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)
    monkeypatch.setattr(
        router,
        "compress",
        lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
            compressed="[C]"
            + c[:20]
            + " <<ccr:t>>",  # recoverable marker so #1307 tool-reversibility guard keeps the compression
            compression_ratio=0.50,
            strategy_used=CompressionStrategy.TEXT,
        ),
    )
    out2 = router.apply(
        [{"role": "tool", "tool_call_id": "t1", "content": content}],
        _ChurnTokenizer(),
        model_limit=1000,
    ).messages[0]["content"]
    assert out2.startswith("[C]"), "once model ready the block must compress"


# ---------------------------------------------------------------------------
# Cache-churn fix — CONTENT-BLOCK path (tool_result blocks).
#
# For Claude Code / Anthropic traffic the prefix is dominated by tool_result
# content-blocks routed through ``_compress_block_content``, not plain-string
# messages. These tests mirror the string-path freeze tests above for that
# dominant path.
# ---------------------------------------------------------------------------


def _tool_result_block_msg(content: str) -> dict:
    """A user message carrying a single ``tool_result`` content-block — the
    Anthropic shape that dominates Claude Code prefixes."""
    return {
        "role": "user",
        "content": [
            {"type": "tool_result", "tool_use_id": "tu1", "content": content},
        ],
    }


def _block_out(router: ContentRouter, content: str, model_limit: int) -> str:
    """Run apply() for a tool_result content-block and return the (possibly
    compressed) string content of that block."""
    result = router.apply(
        [_tool_result_block_msg(content)],
        _ChurnTokenizer(),
        model_limit=model_limit,
    )
    return result.messages[0]["content"][0]["content"]


@pytest.mark.parametrize(
    "suffix",
    ["", '\n```python\nprint("hello")\n```\n', '\n{"records":[{"name":"extra","value":1}]}\n'],
    ids=["status", "fenced-code", "json"],
)
def test_git_status_output_stays_byte_exact_on_direct_and_tool_routes(
    monkeypatch: pytest.MonkeyPatch, suffix: str
) -> None:
    status = """On branch feature/status
Changes to be committed:
  (use "git restore --staged <file>..." to unstage)
\tnew file:   staged path with spaces.txt
\tmodified:   another staged path.txt
	new file:   staged path with several spaces and a longer filename.txt
	modified:   additional staged file.txt

Changes not staged for commit:
  (use "git add <file>..." to update what will be committed)
\tmodified:   edited path with spaces.txt
\tdeleted:    removed path.txt

Untracked files:
  (use "git add <file>..." to include in what will be committed)
\tnew untracked path.txt
"""
    status += suffix
    router = ContentRouter(ContentRouterConfig())

    class LossyProseCompressor:
        def is_ready(self) -> bool:
            return True

        def compress(self, content: str, **_kwargs: object) -> SimpleNamespace:
            compressed = "\n".join(content.splitlines()[::2])
            return SimpleNamespace(compressed=compressed, compressed_tokens=1)

    monkeypatch.setattr(router, "_get_kompress", lambda: LossyProseCompressor())

    direct = router.compress(status)
    tool_result = _block_out(router, status, model_limit=100_000)

    assert direct.compressed == status
    assert tool_result == status


def test_block_freeze_off_is_byte_identical_flapping_baseline(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(b) Block path, flag OFF (default): a mid-zone tool_result block
    (ratio 0.75) flaps — compressed at low pressure, downgraded to passthrough
    once pressure tightens min_ratio below the ratio. Legacy churn preserved."""
    monkeypatch.delenv("HEADROOM_FREEZE_BLOCK_DECISION", raising=False)
    router = _churn_router(monkeypatch, ratio=0.75)
    content = _content_of_n_words(200)

    low = _block_out(router, content, model_limit=100000)
    assert low.startswith("[C]"), "turn1 should compress at low pressure"

    high = _block_out(router, content, model_limit=100)
    assert high == content, "turn2 should flip to passthrough (legacy churn)"
    assert low != high, "flag-off block path must keep flapping (byte change)"


def test_block_freeze_on_pins_compress_verdict_across_turns(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(a) Block path, flag ON: a tool_result block that compresses on first
    sighting keeps the SAME verdict AND SAME bytes across rising-pressure
    turns despite drifting min_ratio. ratio 0.6 < aggressive 0.65."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = _churn_router(monkeypatch, ratio=0.60)
    content = _content_of_n_words(200)

    outs = [_block_out(router, content, model_limit=limit) for limit in (100000, 1000, 300, 100)]
    assert all(o.startswith("[C]") for o in outs), "verdict must stay compress"
    assert len(set(outs)) == 1, "block bytes must be identical across all turns"


def test_block_freeze_on_pins_passthrough_verdict_across_turns(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(a) Block path, flag ON, mid-zone block (ratio 0.75): the RELAXED
    first-sighting threshold (0.85) compresses it on turn 1, freezes the
    compress verdict, and pins it compressed on every later turn even as
    rising pressure pulls min_ratio below 0.75 (where flag-off would
    move_to_skip). Bytes stay compressed/identical and pins are recorded."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = _churn_router(monkeypatch, ratio=0.75)
    content = _content_of_n_words(200)

    outs = [_block_out(router, content, model_limit=limit) for limit in (100000, 1000, 300, 100)]
    assert all(o.startswith("[C]") for o in outs), "verdict must stay compressed"
    assert len(set(outs)) == 1, "block bytes identical (compressed) across turns"
    assert router._freeze_pin_hits > 0, "freeze must pin compress over a tightening min_ratio"


def test_block_model_not_ready_passthrough_is_not_frozen(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Caveat (1) on the block path: a tool_result block that passes through
    ONLY because the ML model is not ready must NOT have its skip verdict
    frozen, so it is re-evaluated once the model is ready."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = ContentRouter(ContentRouterConfig())
    content = _content_of_n_words(200)

    monkeypatch.setattr(router, "_kompress_model_ready", lambda: False)
    monkeypatch.setattr(
        router,
        "compress",
        lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
            compressed=c, compression_ratio=1.0, strategy_used=CompressionStrategy.TEXT
        ),
    )
    out1 = _block_out(router, content, model_limit=1000)
    assert out1 == content, "not-ready -> passthrough"
    assert all(v is True for v in router._frozen_verdicts.values()) or (
        not router._frozen_verdicts
    ), "not-ready block passthrough must not be frozen as skip"

    router._cache.clear()
    monkeypatch.setattr(router, "_kompress_model_ready", lambda: True)
    monkeypatch.setattr(
        router,
        "compress",
        lambda c, context="", bias=1.0, precomputed_detection=None: SimpleNamespace(
            compressed="[C]"
            + c[:20]
            + " <<ccr:t>>",  # recoverable marker so #1307 tool-reversibility guard keeps the compression
            compression_ratio=0.50,
            strategy_used=CompressionStrategy.TEXT,
        ),
    )
    out2 = _block_out(router, content, model_limit=1000)
    assert out2.startswith("[C]"), "once model ready the block must compress"


def test_frozen_verdicts_cleared_on_cache_clear(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(c) ``_frozen_verdicts`` is cleared in lock-step with the cache so it
    cannot outlive the entries it shadows."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = _churn_router(monkeypatch, ratio=0.60)
    content = _content_of_n_words(200)

    _block_out(router, content, model_limit=1000)
    assert router._frozen_verdicts, "a verdict should be frozen after a turn"

    router._cache.clear()
    assert not router._frozen_verdicts, "cache clear must also clear frozen verdicts"


def test_frozen_verdicts_is_size_bounded(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """(c) ``_frozen_verdicts`` is capped with FIFO eviction so it cannot grow
    without bound across a long-lived process."""
    monkeypatch.setenv("HEADROOM_FREEZE_BLOCK_DECISION", "1")
    router = ContentRouter(ContentRouterConfig())
    router._frozen_verdicts_max = 8

    for i in range(50):
        router._record_frozen_verdict(i, True)

    assert len(router._frozen_verdicts) == 8, "store must stay capped at the max"
    # Oldest keys evicted, newest retained (insertion-order FIFO).
    assert set(router._frozen_verdicts) == set(range(42, 50))


def test_frozen_verdict_refuses_unrecoverable_lossy_block() -> None:
    """#1307: a "compress" verdict for a lossy-unmarked block with no CCR
    retrieval marker must NOT be frozen. Pinning it would keep serving an
    unrecoverable summary across turns, which the reversibility guard forbids.
    Recoverable compressions (marked, or a non-lossy strategy) may be pinned.
    """
    router = ContentRouter(ContentRouterConfig())

    # Lossy + no marker -> unrecoverable -> refuse to freeze.
    assert router._frozen_verdict_recoverable(CompressionStrategy.TEXT, "plain summary") is False
    # Lossy + CCR marker -> recoverable -> may freeze.
    assert router._frozen_verdict_recoverable(CompressionStrategy.TEXT, "summary <<ccr:t>>") is True
    # Non-lossy strategy -> recoverable regardless of marker.
    assert (
        router._frozen_verdict_recoverable(CompressionStrategy.SMART_CRUSHER, "no marker") is True
    )
    # Cache-hit path passes the strategy's .value string, not the enum.
    assert router._frozen_verdict_recoverable("text", "plain") is False
    assert router._frozen_verdict_recoverable("text", "x Retrieve more: hash=abc123") is True


def test_detect_timeout_secs_env_parsing(monkeypatch: pytest.MonkeyPatch) -> None:
    """The watchdog budget reads HEADROOM_DETECT_TIMEOUT_SECS; bad values → default."""
    get = content_router_module._detect_timeout_secs
    default = content_router_module._DEFAULT_DETECT_TIMEOUT_SECS

    monkeypatch.delenv("HEADROOM_DETECT_TIMEOUT_SECS", raising=False)
    assert get() == default

    monkeypatch.setenv("HEADROOM_DETECT_TIMEOUT_SECS", "0.25")
    assert get() == 0.25

    monkeypatch.setenv("HEADROOM_DETECT_TIMEOUT_SECS", "nope")
    assert get() == default

    monkeypatch.setenv("HEADROOM_DETECT_TIMEOUT_SECS", "0")
    assert get() == default


def test_rust_detect_watchdog_passes_through_result() -> None:
    """A fast native detector returns its result unchanged through the watchdog."""
    sentinel = SimpleNamespace(content_type="json_array", confidence=1.0, metadata={})
    out = content_router_module._rust_detect_watchdogged(lambda _content: sentinel, "payload", 5.0)
    assert out is sentinel


def test_rust_detect_watchdog_relays_native_error() -> None:
    """An exception raised inside the native detector propagates to the caller."""

    def boom(_content: str) -> None:
        raise ValueError("native boom")

    with pytest.raises(ValueError, match="native boom"):
        content_router_module._rust_detect_watchdogged(boom, "payload", 5.0)


def test_detect_content_watchdog_degrades_on_windows_hang(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """A hung native detect on Windows degrades to pure-Python, never deadlocks (#575)."""
    import threading as _threading

    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
    monkeypatch.setattr(content_router_module.sys, "platform", "win32")
    monkeypatch.setenv("HEADROOM_DETECT_TIMEOUT_SECS", "0.1")

    release = _threading.Event()

    def _hang(_content: str):
        release.wait()  # simulate the WaitOnAddress park (GIL released while waiting)
        return SimpleNamespace(content_type="plain_text", confidence=1.0, metadata={})

    monkeypatch.setattr(_core, "detect_content_type", _hang)

    try:
        # JSON content: the pure-Python regex fallback recognizes it as JSON_ARRAY,
        # proving we took the degrade path rather than the (hung) native result.
        result = _detect_content('[{"id": 1}]')
        assert result.content_type is ContentType.JSON_ARRAY
    finally:
        release.set()  # let the daemon worker finish so it does not linger


def test_detect_content_watchdog_uses_native_result_on_windows(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """On Windows with rust forced, a fast native result still flows through unchanged."""
    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
    monkeypatch.setattr(content_router_module.sys, "platform", "win32")

    fake = SimpleNamespace(content_type="source_code", confidence=1.0, metadata={})
    monkeypatch.setattr(_core, "detect_content_type", lambda _content: fake)

    result = _detect_content("def main(): pass")
    assert result.content_type is ContentType.SOURCE_CODE


def test_detect_content_circuit_breaker_skips_native_after_hang(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """After one watchdog timeout, native detection is disabled process-wide (#575)."""
    import threading as _threading

    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
    monkeypatch.setattr(content_router_module.sys, "platform", "win32")
    monkeypatch.setenv("HEADROOM_DETECT_TIMEOUT_SECS", "0.1")

    release = _threading.Event()
    calls = 0

    def _hang(_content: str):
        nonlocal calls
        calls += 1
        release.wait()  # park with GIL released, like the real WaitOnAddress hang
        return SimpleNamespace(content_type="plain_text", confidence=1.0, metadata={})

    monkeypatch.setattr(_core, "detect_content_type", _hang)
    try:
        first = _detect_content('[{"id": 1}]')
        second = _detect_content('[{"id": 2}]')
        assert first.content_type is ContentType.JSON_ARRAY
        assert second.content_type is ContentType.JSON_ARRAY
        assert calls == 1  # breaker tripped: native entered once, 2nd call skipped it
    finally:
        release.set()  # let the lone daemon worker finish


def test_strip_detection_envelope_isolates_tool_output_payload() -> None:
    """Only a whole-string tool-output envelope is unwrapped; content that
    merely mentions the tags, or has an empty body, is left untouched."""
    body = "def main():\n    return 1"
    wrapped = f"<returncode>0</returncode>\n<output>\n{body}\n</output>"
    assert _strip_detection_envelope(wrapped) == body
    # <output> alias tags and a bare envelope (no returncode) also unwrap.
    assert _strip_detection_envelope(f"<stdout>\n{body}\n</stdout>") == body
    # Non-envelope content is returned verbatim (no "<" fast-path + no match).
    prose = "see the <output> tag docs for details"
    assert _strip_detection_envelope(prose) == prose
    # Empty body never yields an empty probe — falls back to the original.
    empty = "<output>\n\n</output>"
    assert _strip_detection_envelope(empty) == empty


def test_detect_content_sees_through_tool_output_envelope() -> None:
    """Regression: a tool-result envelope's tags used to make the detector
    read the whole payload as markup and misroute code to the HTML extractor.
    Detection now runs on the inner payload, so the real type wins."""
    code = "\n".join(
        [
            "import os",
            "from pathlib import Path",
            "",
            "def main() -> int:",
            "    return len(os.listdir(Path.cwd()))",
        ]
    )
    wrapped = f"<returncode>0</returncode>\n<output>\n{code}\n</output>"
    assert _detect_content(wrapped).content_type is ContentType.SOURCE_CODE
    assert _detect_content(wrapped).content_type is _detect_content(code).content_type


def test_detect_content_overrides_html_misroute_for_grep_and_logs(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Regression: the native detector (magika) tags dense grep output and
    build logs as HTML because file paths and </> read as markup. Routing those
    to the HTML article-extractor is lossy (it strips code + identifiers). When
    the structural log/search detectors positively claim the payload they
    override the HTML verdict (log checked first so tracebacks win); genuine
    HTML with no such structure is left as HTML."""
    import headroom._core as _core

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")
    monkeypatch.setattr(
        _core,
        "detect_content_type",
        lambda content: SimpleNamespace(content_type="html", confidence=1.0, metadata={}),
    )

    # grep over HTML template files: native says html, but it is search results.
    grep = "\n".join(
        f'templates/pages/dashboard_{i}.html:{10 + i}:      <div class="card" data-id="{i}">'
        for i in range(6)
    )
    assert _detect_content(grep).content_type is ContentType.SEARCH_RESULTS

    # build/error log misread as html -> LOG wins (checked before search).
    build_log = "\n".join(
        [
            "ERROR failed to compile module widget",
            "WARNING deprecated call near <template>",
            "Traceback (most recent call last):",
            "ERROR build aborted after 2 retries",
        ]
    )
    assert _detect_content(build_log).content_type is ContentType.BUILD_OUTPUT

    # genuine HTML article: no grep/log structure -> override does not fire.
    html = (
        "<!DOCTYPE html>\n<html><head><title>x</title></head>"
        "<body><main><section><p>An article about widgets and gadgets.</p>"
        "</section></main></body></html>"
    )
    assert _detect_content(html).content_type is ContentType.HTML


def test_datetime_prefixed_user_prompt_survives_router() -> None:
    """Regression (2026-08-23): interactive wrap-copilot prompts were deleted.

    Copilot CLI prepends ``<current_datetime>…</current_datetime>`` to every
    interactive user turn; the ISO timestamp matched the grep ``file:line:``
    detector, the one-line prompt classified as SEARCH_RESULTS, and
    SearchCompressor kept only the datetime line — the model received no
    request and answered "How can I help you today?". The router must never
    route this shape to the search line-filter and must keep the prose.
    """
    prompt = (
        "<current_datetime>2026-08-23T09:57:59.792+02:00</current_datetime>\n"
        "\n"
        "Please update the PR desc and check .overlay/ for hints."
    )
    result = ContentRouter().compress(prompt)
    assert result.strategy_used is not CompressionStrategy.SEARCH
    assert "Please update the PR desc" in result.compressed


# --- The caller's prompt stays verbatim on replaying paths -----------------
#
# The proxy's coding profile turns user-message compression on so tool
# observations inside user messages shrink. On a replaying path (proxy
# handlers, the /v1/compress session turn) that used to reach the prompt text
# too, and only a cache_control marker stopped it: Claude Code sets one, a
# plain agent does not, and its task statement went out with stop words
# stripped and pytest node ids mangled.

_TASK = "Fix this bug in the library source. " * 60


def _prompt_router(monkeypatch: pytest.MonkeyPatch) -> tuple[ContentRouter, list[str]]:
    return _fresh_cc_router(monkeypatch)


def test_opening_task_text_block_is_verbatim_under_replay(monkeypatch: pytest.MonkeyPatch) -> None:
    router, _ = _prompt_router(monkeypatch)
    messages = [{"role": "user", "content": [{"type": "text", "text": _TASK}]}]
    out = router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert out.messages[0]["content"][0]["text"] == _TASK


def test_opening_task_string_is_verbatim_under_replay(monkeypatch: pytest.MonkeyPatch) -> None:
    router, _ = _prompt_router(monkeypatch)
    messages = [{"role": "user", "content": _TASK}]
    out = router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert out.messages[0]["content"] == _TASK


def test_newest_user_turn_keeps_text_but_compresses_tool_results(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    router, calls = _prompt_router(monkeypatch)
    follow_up = "Now also handle the async path. " * 60
    messages = [
        {"role": "user", "content": "run the tests"},
        {
            "role": "assistant",
            "content": [{"type": "tool_use", "id": "t1", "name": "bash", "input": {}}],
        },
        {
            "role": "user",
            "content": [
                {"type": "tool_result", "tool_use_id": "t1", "content": _FRESH_CC_OUTPUT},
                {"type": "text", "text": follow_up},
            ],
        },
    ]
    out = router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    blocks = out.messages[2]["content"]
    assert blocks[0]["content"].endswith("[compressed]")
    assert blocks[1]["text"] == follow_up
    assert follow_up not in calls


def test_text_harness_observation_string_still_compresses_under_replay(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # A text harness returns its tool output as a role:user string after the
    # assistant turn. That is an observation, not the prompt.
    router, calls = _prompt_router(monkeypatch)
    messages = [
        {"role": "user", "content": "run the tests"},
        {"role": "assistant", "content": "```bash\npytest -q\n```"},
        {"role": "user", "content": _FRESH_CC_OUTPUT},
    ]
    router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert _FRESH_CC_OUTPUT in calls


def test_follow_up_prompt_string_after_assistant_turn_is_verbatim_under_replay(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # A plain-string follow-up after an assistant reply is the caller's new
    # instruction, the same as the list-content case above.
    router, calls = _prompt_router(monkeypatch)
    follow_up = "Now also handle the async path in tenacity/asyncio. " * 60
    messages = [
        {"role": "user", "content": "run the tests"},
        {"role": "assistant", "content": "All 167 tests pass on the sync path."},
        {"role": "user", "content": follow_up},
    ]
    out = router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert out.messages[2]["content"] == follow_up
    assert follow_up not in calls


def test_text_harness_observation_text_block_still_compresses_under_replay(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    # The list-content twin of the string observation above: both shapes
    # decide "prompt or observation" the same way.
    router, calls = _prompt_router(monkeypatch)
    messages = [
        {"role": "user", "content": "run the tests"},
        {"role": "assistant", "content": "```bash\npytest -q\n```"},
        {"role": "user", "content": [{"type": "text", "text": _FRESH_CC_OUTPUT}]},
    ]
    router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        prefix_replay_guaranteed=True,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert _FRESH_CC_OUTPUT in calls


def test_prompt_text_compresses_without_replay_guarantee(monkeypatch: pytest.MonkeyPatch) -> None:
    # SDK / document callers that opt into user compression keep it: a
    # spreadsheet or pasted document in a user message is the payload.
    router, calls = _prompt_router(monkeypatch)
    messages = [{"role": "user", "content": [{"type": "text", "text": _TASK}]}]
    router.apply(
        messages,
        _ChurnTokenizer(),
        model_limit=100_000,
        compress_user_messages=True,
        min_tokens_to_compress=1,
    )
    assert _TASK in calls


def _html_doc_with_embedded_json(*, padded: bool = True) -> str:
    """A whole HTML document with one embedded JSON array-of-objects.

    The JSON sits in a single-line ``<script>`` so the mixed-content regex
    heuristics do not fire: the block detects as pure HTML and takes the
    ``_compress_pure`` path. ``padded`` keeps the default ``json.dumps``
    separators so the array is minifiable — a compact array gives the JSON
    route no benefit and both passes no-op.
    """
    rows = [
        {
            "id": i,
            "name": f"item-{i}",
            "tags": ["alpha", "beta"],
            "score": i * 3,
            "description": "a routine record with repeated descriptive fields",
        }
        for i in range(60)
    ]
    embedded = json.dumps(rows) if padded else json.dumps(rows, separators=(",", ":"))
    paras = "".join(
        f"<p>Paragraph {i}: distributed systems rely on consensus protocols to "
        f"agree on state. Raft separates leader election from log replication, "
        f"which makes it easier to reason about.</p>\n"
        for i in range(30)
    )
    return (
        "<!DOCTYPE html>\n<html><head><title>Raft Notes</title></head>\n<body>\n"
        "<article><h1>Raft consensus notes</h1>\n"
        + paras
        + '<script type="application/json">'
        + embedded
        + "</script>\n"
        + paras
        + "</article>\n</body></html>\n"
    )


def test_html_with_embedded_json_runs_extractor_not_json_shortcut() -> None:
    """#3609: the embedded-JSON pre-pass must not short-circuit HTML extraction.

    Before the fix, a small local JSON minify (~10% win) returned immediately
    from ``_apply_strategy_to_content`` and whole-document extraction (~36%
    win on this document) never ran; the issue reported the same shape at
    2% vs 95.5%. The splice is now deferred for HTML: extraction gets first
    refusal and the JSON route only fires when extraction finds nothing.
    """
    pytest.importorskip("trafilatura")
    doc = _html_doc_with_embedded_json()
    assert _detect_content(doc).content_type is ContentType.HTML
    assert not is_mixed_content(doc)

    result = ContentRouter().compress(doc)

    assert result.strategy_used is CompressionStrategy.HTML
    assert result.strategy_chain == ["html"]
    # extraction ran: markup is gone and the document collapsed hard
    # (the pre-fix JSON shortcut left <html> in place at ~10% reduction)
    assert "<html>" not in result.compressed
    assert len(result.compressed) < len(doc) * 3 // 5


def test_json_in_html_preserves_records_and_markup() -> None:
    rows = [
        {"id": i, "account": f"ACCOUNT_{i}", "score": i * 13, "active": i % 2 == 0}
        for i in range(60)
    ]
    prefix = (
        "<!DOCTYPE html>\n<html><head><title>data</title></head>\n<body>\n"
        '<script type="application/json">'
    )
    suffix = "</script>\n</body></html>\n"
    shell = prefix + json.dumps(rows) + suffix
    router = ContentRouter(
        ContentRouterConfig(
            lossless=False,
            enable_html_extractor=True,
            smart_crusher_lossless_only=True,
            smart_crusher_with_compaction=False,
            enable_kompress=False,
        )
    )

    result = router.compress(shell)

    assert result.compressed.startswith(prefix)
    assert result.compressed.endswith(suffix)
    assert json.loads(result.compressed[len(prefix) : -len(suffix)]) == rows
    assert result.tokens_saved > 0
    assert not result.unrecoverable_sections


# A Claude Code Read of Rust: line-numbered code that code_aware hands back
# unchanged (it cannot parse the number column) while reporting compressed=True.
# Code runs ~2 tokens per whitespace word, so a word count badly undercounts it.
_NUMBERED_RUST = "\n".join(
    f"{i}\tlet value_{i} = compute(&items[{i}..], Some(opts.clone()))?;" for i in range(1, 121)
)


def _noop_code_aware_router(monkeypatch: pytest.MonkeyPatch, kompress_out: str) -> ContentRouter:
    monkeypatch.delenv("HEADROOM_LOSSLESS_THEN_LOSSY", raising=False)
    router = ContentRouter(ContentRouterConfig(enable_code_aware=True))

    class NoopCodeCompressor:
        def compress(self, content: str, language=None, context: str = "") -> SimpleNamespace:
            return SimpleNamespace(compressed=content, compressed_tokens=0)

    monkeypatch.setattr(router, "_get_code_compressor", lambda: NoopCodeCompressor())

    def kompress(content: str, *_args: object, **_kwargs: object) -> tuple[str, int]:
        # What Kompress reports: payload tokens for a real compression, the
        # WORD count for a passthrough.
        if kompress_out == content:
            return content, len(content.split())
        return kompress_out, _estimate_tokens(kompress_out)

    monkeypatch.setattr(router, "_try_ml_compressor", kompress)
    return router


def test_code_aware_noop_keeps_a_smaller_kompress_fallback(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """The fallback is judged in tokens, the unit of the code_aware result.

    With code_aware's result counted in words, a Kompress result at ~60% of the
    tokens still "lost" to the unchanged block, so the inference always ran and
    was always thrown away.
    """
    kompressed = "\n".join(f"{i}\tvalue_{i} compute(&items[{i}..])" for i in range(1, 121))
    assert _estimate_tokens(kompressed) < _estimate_tokens(_NUMBERED_RUST)
    assert _estimate_tokens(kompressed) > len(_NUMBERED_RUST.split())
    router = _noop_code_aware_router(monkeypatch, kompressed)

    compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
        _NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
    )

    assert compressed == kompressed
    assert compressed_tokens == _estimate_tokens(kompressed)
    assert strategy_chain == ["code_aware", "kompress"]


def test_code_aware_noop_reports_tokens_when_kompress_passes_through(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """An unchanged block is reported at its own token count, never its word count."""
    router = _noop_code_aware_router(monkeypatch, _NUMBERED_RUST)

    compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(
        _NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
    )

    assert compressed == _NUMBERED_RUST
    assert compressed_tokens == _estimate_tokens(_NUMBERED_RUST)


def test_code_aware_noop_runs_kompress_once_when_it_passes_through(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Lossless-then-lossy tries Kompress inline; a passthrough must not be retried."""
    router = _noop_code_aware_router(monkeypatch, _NUMBERED_RUST)
    router._lossless_then_lossy = True
    calls: list[str] = []
    kompress = router._try_ml_compressor

    def counting(content: str, *args: object, **kwargs: object) -> tuple[str, int]:
        calls.append(content)
        return kompress(content, *args, **kwargs)

    monkeypatch.setattr(router, "_try_ml_compressor", counting)

    compressed, compressed_tokens, _chain, _reversible = router._apply_strategy_to_content(
        _NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
    )

    assert compressed == _NUMBERED_RUST
    assert compressed_tokens == _estimate_tokens(_NUMBERED_RUST)
    assert len(calls) == 1


def test_code_aware_noop_keeps_a_smaller_inline_kompress(
    monkeypatch: pytest.MonkeyPatch,
) -> None:
    """Lossless-then-lossy's inline attempt keeps a Kompress result that wins.

    Measured in tokens, a code_aware no-op no longer looks like a shrink, so the
    inline attempt fires and its smaller result is kept without a second call.
    """
    kompressed = "\n".join(f"{i}\tvalue_{i} compute(&items[{i}..])" for i in range(1, 121))
    router = _noop_code_aware_router(monkeypatch, kompressed)
    router._lossless_then_lossy = True
    calls: list[str] = []
    kompress = router._try_ml_compressor

    def counting(content: str, *args: object, **kwargs: object) -> tuple[str, int]:
        calls.append(content)
        return kompress(content, *args, **kwargs)

    monkeypatch.setattr(router, "_try_ml_compressor", counting)

    compressed, compressed_tokens, strategy_chain, _reversible = router._apply_strategy_to_content(
        _NUMBERED_RUST, CompressionStrategy.CODE_AWARE, context=""
    )

    assert compressed == kompressed
    assert compressed_tokens == _estimate_tokens(kompressed)
    assert strategy_chain == ["code_aware", "kompress"]
    assert len(calls) == 1


@pytest.mark.parametrize("as_block", [False, True], ids=["string-tool", "tool-result-block"])
@pytest.mark.parametrize("other_marker", [False, True], ids=["no-marker", "unrelated-marker"])
@pytest.mark.parametrize("ccr_enabled", [False, True], ids=["explicit-no-ccr", "reversible-tools"])
def test_mixed_lossy_section_cannot_borrow_another_sections_retrieval_marker(
    monkeypatch, as_block, other_marker, ccr_enabled
) -> None:
    router = ContentRouter(
        ContentRouterConfig(
            enable_code_aware=True,
            enable_kompress=False,
            min_chars_for_block_compression=0,
            protect_recent_code=0,
            protect_analysis_context=False,
            ccr_inject_marker=ccr_enabled,
        )
    )
    code = (
        "def account():\n"
        + "".join(
            f"    # audit record {i}: preserve the complete account history.\n" for i in range(120)
        )
        + '    identifier = "ACCOUNT_ID_42"\n    return identifier\n'
    )
    content = "```python\n" + code + "```\n\n" + '[{"flag":true,"value":7}]'
    if other_marker:
        content += "\n\nIndependent retained section <<ccr:123456789012>>"

    def compress_section(section, strategy, *_args, **_kwargs):
        # Control only the lossy compressor boundary; split/routing/guards are real.
        compressed = (
            section.replace("ACCOUNT_ID_42", "")
            if strategy is CompressionStrategy.CODE_AWARE
            else section
        )
        return compressed, _estimate_tokens(compressed), [strategy.value], compressed == section

    monkeypatch.setattr(router, "_apply_strategy_to_content", compress_section)
    monkeypatch.setattr(
        router,
        "compress",
        lambda c, context="", bias=1.0, precomputed_detection=None: router._compress_mixed(
            c, context, bias=bias
        ),
    )
    direct = router.compress(content)
    assert direct.strategy_used is CompressionStrategy.MIXED
    assert direct.compression_ratio < 1.0
    assert "ACCOUNT_ID_42" not in direct.compressed

    message = (
        _tool_result_block_msg(content)
        if as_block
        else {"role": "tool", "tool_call_id": "account", "content": content}
    )
    for _turn in range(2):
        result = router.apply([message], _ChurnTokenizer(), model_limit=100_000)
        output = (
            result.messages[0]["content"][0]["content"]
            if as_block
            else result.messages[0]["content"]
        )
        if ccr_enabled or other_marker:
            assert output == content
        else:
            assert "ACCOUNT_ID_42" not in output
            assert '[{"flag":true,"value":7}]' in output


@pytest.mark.parametrize("as_block", [False, True], ids=["string-tool", "tool-result-block"])
@pytest.mark.parametrize("mode", ["minify", "unmarked-loss", "sibling-marker"])
def test_embedded_json_mixed_tool_output_preserves_every_record(
    monkeypatch: pytest.MonkeyPatch, as_block: bool, mode: str
) -> None:
    from headroom.providers import OpenAIProvider
    from headroom.tokenizer import Tokenizer

    monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "python")
    router = ContentRouter(
        ContentRouterConfig(
            smart_crusher_with_compaction=False,
            smart_crusher_max_items_after_crush=1000,
            enable_kompress=False,
            lossless_then_lossy=False,
            relevance_split=False,
            min_chars_for_block_compression=0,
            protect_recent_code=0,
            protect_analysis_context=False,
            ccr_inject_marker=True,
        )
    )
    rows = [{"id": i, "account": f"ACCOUNT_ID_{i}", "value": i * 13} for i in range(60)]
    fenced = "\n".join(f"Retain the complete diagnostic note for customer {i}." for i in range(30))
    content = f"```text\n{fenced}\n```\n\n"
    if mode == "sibling-marker":
        sibling = json.dumps([{"recover": i} for i in range(20)])
        content += f"Independent: {sibling}\n\n"
    content += f"Records: {json.dumps(rows)}\n\nDone."
    if mode != "minify":
        # Control the lossy compressor only. Nested dispatch, span adoption,
        # mixed routing, both safety guards and their caches remain production code.
        def crush(span, **_kwargs):
            values = json.loads(span)
            text = (
                "<<ccr:123456789012>>"
                if "recover" in values[0]
                else json.dumps(values[:1], separators=(",", ":"))
            )
            return SimpleNamespace(compressed=text)

        monkeypatch.setattr(router, "_get_smart_crusher", lambda: SimpleNamespace(crush=crush))

    tokenizer = Tokenizer(OpenAIProvider().get_token_counter("gpt-4o"), "gpt-4o")
    message = (
        _tool_result_block_msg(content)
        if as_block
        else {"role": "tool", "tool_call_id": "records", "content": content}
    )
    for _turn in range(2):  # Cold compression followed by the cached/skip verdict.
        result = router.apply([message], tokenizer, model_limit=100_000)
        output = (
            result.messages[0]["content"][0]["content"]
            if as_block
            else result.messages[0]["content"]
        )
        if mode == "minify":
            assert tokenizer.count_text(output) < tokenizer.count_text(content)
            assert f"```text\n{fenced}\n```" in output
            assert output.endswith("Done.")
            assert "<<ccr:" not in output
            recovered, _end = json.JSONDecoder().raw_decode(output.split("Records: ", 1)[1])
            assert recovered == rows
        else:
            assert output == content