Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(ci): prevent native detector from hanging test shards (#2996)
JD Davisa708c0571eec
diff --git a/headroom/observability/metrics.py b/headroom/observability/metrics.pyindex d0b51745e..22d5da309 100644--- a/headroom/observability/metrics.py+++ b/headroom/observability/metrics.py@@ -227,9 +227,7 @@ class HeadroomOtelMetrics:)self._proxy_attempted_input_tokens = self._meter.create_counter("headroom.proxy.tokens.attempted_input",- description=(- "Input tokens Headroom attempted to optimize before compression."- ),+ description=("Input tokens Headroom attempted to optimize before compression."),unit="1",)self._proxy_output_saved_tokens = self._meter.create_counter(@@ -239,9 +237,7 @@ class HeadroomOtelMetrics:)self._proxy_savings_usd = self._meter.create_counter("headroom.proxy.savings.usd",- description=(- "Estimated savings in USD by distinct Headroom or provider-cache layer."- ),+ description=("Estimated savings in USD by distinct Headroom or provider-cache layer."),unit="USD",)self._proxy_saved_tokens = self._meter.create_counter(diff --git a/headroom/proxy/savings_tracker.py b/headroom/proxy/savings_tracker.pyindex 68e821154..4c10cac6b 100644--- a/headroom/proxy/savings_tracker.py+++ b/headroom/proxy/savings_tracker.py@@ -14,12 +14,12 @@ import mathimport osimport tempfileimport threading+from collections.abc import Mappingfrom csv import DictWriterfrom datetime import datetime, timedelta, timezonefrom functools import lru_cachefrom io import StringIOfrom pathlib import Path-from collections.abc import Mappingfrom typing import Anyfrom headroom import paths as _pathsdiff --git a/headroom/transforms/content_router.py b/headroom/transforms/content_router.pyindex fd0704067..0ba64cb17 100644--- a/headroom/transforms/content_router.py+++ b/headroom/transforms/content_router.py@@ -98,7 +98,6 @@ split_into_sections = _mixed_content.split_into_sections_detect_backend_warned = False_detect_panic_warned = False_detect_native_unhealthy = False # circuit breaker: native detect hung once (#575)-_detect_native_verified = False # native detect has returned once -> skip the watchdog# Shared calibrated fallback estimator (tiktoken cl100k_base ~90% accuracy,@@ -917,7 +916,6 @@ def _detect_content(content: str) -> DetectionResult:`_strategy_from_detection` keys off that field alone."""global _detect_backend_warned, _detect_panic_warned, _detect_native_unhealthy- global _detect_native_verified# Detect on the unwrapped payload so a tool-output envelope's tags don't get# the whole result misclassified as HTML/XML (#route-converter corruption).@@ -957,19 +955,13 @@ def _detect_content(content: str) -> DetectionResult:from headroom._core import detect_content_type as _rust_detecttry:- # The native detector can deadlock on FIRST use (#575 — seen on Windows- # and macOS/arm64). Bound it with a watchdog so a hang degrades to the- # pure-Python detector; the previous win32-only guard left other- # platforms unprotected, so a hung Linux sidecar silently stopped- # compressing (every request failed open to passthrough). Watchdog until- # the native detector has returned once, then use the direct fast path —- # the hang is first-use only, so steady state pays no per-call thread- # overhead. win32 keeps watchdogging every call (unchanged).- if sys.platform == "win32" or not _detect_native_verified:- rust_result = _rust_detect_watchdogged(_rust_detect, content, _detect_timeout_secs())- else:- rust_result = _rust_detect(content)- _detect_native_verified = True # returned without hanging -> trusted hot path+ # Native detector state can become wedged after an earlier successful+ # call (for example when another test or component initializes ORT).+ # A one-time "verified" fast path therefore turns a later native stall+ # into an unbounded process hang. Keep every call bounded; on timeout+ # the process-wide circuit breaker below makes subsequent calls use the+ # pure-Python detector without spawning more watchdog threads.+ rust_result = _rust_detect_watchdogged(_rust_detect, content, _detect_timeout_secs())# Rust's `content_type` is the lowercase string tag (e.g.# "json_array"); translate to the Python `ContentType` enum so# downstream mapping keys match.diff --git a/tests/test_codex_ws_compression_scheduler.py b/tests/test_codex_ws_compression_scheduler.pyindex 69b842285..7561c9b2b 100644--- a/tests/test_codex_ws_compression_scheduler.py+++ b/tests/test_codex_ws_compression_scheduler.py@@ -300,30 +300,14 @@ def test_concurrent_compression_has_no_semaphore_tail() -> None:)assert not errors, f"Got {len(errors)} errors; first: {errors[0].error}"- ratio = p99 / max(p50, 1)SEMAPHORE_P99_CEILING_MS = 1_000.0assert p99 < SEMAPHORE_P99_CEILING_MS, (f"p99 is {p99:.0f}ms; expected < {SEMAPHORE_P99_CEILING_MS:.0f}ms on ""uniform-size workload. The pre-fix semaphore baseline was ~2433ms.")- # The p99/p50 ratio only signals contention when the tail is also- # *absolutely* large. On a fast/quiet runner p50 rounds toward 0ms, so the- # ratio collapses to "p99 in ms" and a few milliseconds of ordinary- # scheduler jitter reads as a spurious multiple (e.g. p50=0ms, p99=5ms →- # ~5×) that has nothing to do with the semaphore. The deleted semaphore- # produced a tail of *tens* of milliseconds (and ~27×); a healthy run keeps- # p99 in the single-digit-ms range regardless of ratio. So only treat a high- # ratio as a regression once p50 is measurable and p99 clears a noise floor.- # Hosted CI can occasionally park one worker for a few dozen milliseconds- # even when the compression path is healthy; the semaphore regression this- # test guards against had a seconds-scale p99 and is still bounded by the- # hard p99 guard above.- SEMAPHORE_TAIL_FLOOR_MS = 75.0- assert p50 < 1.0 or ratio < 4.0 or p99 < SEMAPHORE_TAIL_FLOOR_MS, (- f"p99/p50 ratio is {ratio:.1f}× (p50={p50:.0f}ms, p99={p99:.0f}ms). "- f"Expected < 4× on uniform-size workload once p50 is measurable and p99 clears "- f"the {SEMAPHORE_TAIL_FLOOR_MS:.0f}ms noise floor — a high ratio with a large "- f"absolute tail means the semaphore-induced contention tail may be back. "- f"Pre-fix baseline ratio on this same workload shape was ~27× regardless "- f"of CPU speed."- )+ # Do not add a p99/p50 wall-clock ratio here. A hosted runner can park one+ # worker independently of this code path, making an otherwise healthy+ # 2ms/76ms distribution look like a 35x contention tail. The property is+ # covered structurally above (the semaphore and nested executor must stay+ # absent), while this absolute ceiling still rejects the measured 2433ms+ # pre-fix behavior without pretending scheduler jitter is product state.diff --git a/tests/test_transforms/test_detect_fallback_1123.py b/tests/test_transforms/test_detect_fallback_1123.pyindex aa6b0036e..22ad4f72a 100644--- a/tests/test_transforms/test_detect_fallback_1123.py+++ b/tests/test_transforms/test_detect_fallback_1123.py@@ -20,8 +20,9 @@ from headroom.transforms import content_router as cr@pytest.fixture(autouse=True)def _compatible_mock_native_runtime(monkeypatch: pytest.MonkeyPatch) -> None:- """These tests replace the native detector, so keep the ORT preflight open."""+ """Keep mocked native calls reachable regardless of prior test state."""monkeypatch.setattr(ort_runtime, "rust_ort_runtime_compatible", lambda: True)+ monkeypatch.setattr(cr, "_detect_native_unhealthy", False)def test_falls_back_on_rust_exception(monkeypatch):diff --git a/tests/test_transforms_content_router.py b/tests/test_transforms_content_router.pyindex aeaf48aaf..4f76c6386 100644--- a/tests/test_transforms_content_router.py+++ b/tests/test_transforms_content_router.py@@ -1,6 +1,7 @@from __future__ import annotationsimport json+import threadingfrom types import SimpleNamespaceimport pytest@@ -160,6 +161,29 @@ def test_content_signature_and_detection_helpers(monkeypatch: pytest.MonkeyPatchassert result.metadata == {}+def test_native_detection_remains_bounded_after_success(monkeypatch: pytest.MonkeyPatch) -> None:+ """A successful native call must not disable the watchdog for later calls."""+ import headroom._core as _core++ monkeypatch.setenv("HEADROOM_DETECT_BACKEND", "rust")+ monkeypatch.setattr(content_router_module, "_detect_timeout_secs", lambda: 0.01)+ calls = 0++ def _succeeds_then_hangs(_content: str) -> SimpleNamespace:+ nonlocal calls+ calls += 1+ if calls == 1:+ return SimpleNamespace(content_type="plain_text")+ threading.Event().wait()+ raise AssertionError("unreachable")++ monkeypatch.setattr(_core, "detect_content_type", _succeeds_then_hangs)++ assert _detect_content("first").content_type is ContentType.PLAIN_TEXT+ assert _detect_content("second").content_type is ContentType.PLAIN_TEXT+ assert content_router_module._detect_native_unhealthy is True++def test_mixed_content_section_splitting_and_json_extraction() -> None:content = "\n".join([