Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
deps: Bump lru from 0.18.5 to 0.19.0 (#4382)
dependabot[bot]038af599e1a3
diff --git a/Cargo.lock b/Cargo.lockindex 24e4b1d4d..cf9d8c5d1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -2710,9 +2710,9 @@ dependencies = [[[package]]name = "lru"-version = "0.18.5"+version = "0.19.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ef9ac18847474e638e3702b76c65d4eb93428471a74778ef0f1be711717f89b5"+checksum = "4a9f71e3591c7ec44d3e563e87bf14ddc2bada8519dca046e5b0e560cd004e2d"dependencies = ["hashbrown 0.17.1",]diff --git a/crates/headroom-proxy/Cargo.toml b/crates/headroom-proxy/Cargo.tomlindex 686b8b830..52e891110 100644--- a/crates/headroom-proxy/Cargo.toml+++ b/crates/headroom-proxy/Cargo.toml@@ -91,7 +91,11 @@ sha2 = "0.11"# unbounded memory growth from a flood of unique session keys. `lru`# is the de-facto Rust LRU crate; minimal surface, no dependencies of# our own beyond `hashbrown` (which we already pull transitively).-lru = "0.18"+# 0.19 corrects the Send bound on shared iterators to require Sync: shared+# references may cross threads only when their referents are Sync. Our caches+# use String/Vec<String>/StructuralHash behind Arc<Mutex<_>> and do not+# transfer iterators; retain the upstream soundness fix without API changes.+lru = "0.19"# PR-D4: GCP Application Default Credentials → bearer token for# Vertex `:rawPredict` / `:streamRawPredict`. See workspace# Cargo.toml for rationale.diff --git a/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs b/crates/headroom-proxy/src/cache_stabilization/drift_detector.rsindex 12ca1f709..c3fe1c884 100644--- a/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs+++ b/crates/headroom-proxy/src/cache_stabilization/drift_detector.rs@@ -66,7 +66,7 @@//! key-sorted rebuild) of the corresponding subtree. Still well//! under a millisecond on a typical agentic request; the detector//! stays log-only and off the forwarded-bytes path.-//! - One LRU lookup + insert. `lru = "0.12"` is O(1) amortised.+//! - One LRU lookup + insert. `lru` is O(1) amortised.//! - One `tracing::info!` or `tracing::warn!`. No metric emission yet//! (left for Phase F PR-F* when the global Prometheus registry can//! accept session-scoped counters without a cardinality explosion).diff --git a/scripts/verify_native_session_caches.py b/scripts/verify_native_session_caches.pynew file mode 100644index 000000000..db9a094f3--- /dev/null+++ b/scripts/verify_native_session_caches.py@@ -0,0 +1,201 @@+"""Exercise native session caches against a credentialless loopback upstream.++Build: cargo build -p headroom-proxy --locked+Run: python scripts/verify_native_session_caches.py --proxy target/debug/headroom-proxy+On Windows append .exe. A custom CARGO_TARGET_DIR requires its binary path.+Only Python's standard library is required. Logs are temporary unless --log is set.+"""++import argparse+import concurrent.futures+import http.server+import json+import os+import socket+import subprocess+import tempfile+import threading+import time+import urllib.request+from pathlib import Path+++def main():+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--proxy", required=True, type=Path)+ parser.add_argument("--log", type=Path, help="preserve native proxy logs here")+ args = parser.parse_args()+ binary = args.proxy.resolve(strict=True)+ captured = []+ capture_lock = threading.Lock()++ class Upstream(http.server.BaseHTTPRequestHandler):+ def do_POST(self):+ body = self.rfile.read(int(self.headers["Content-Length"]))+ with capture_lock:+ captured.append((json.loads(body), self.headers.get("anthropic-beta", "")))+ reply = json.dumps(+ {+ "id": "local",+ "type": "message",+ "role": "assistant",+ "model": "claude-sonnet-4-20250514",+ "content": [{"type": "text", "text": "local stub"}],+ "stop_reason": "end_turn",+ "usage": {"input_tokens": 10, "output_tokens": 2},+ }+ ).encode()+ self.send_response(200)+ self.send_header("Content-Type", "application/json")+ self.send_header("Content-Length", str(len(reply)))+ self.end_headers()+ self.wfile.write(reply)++ def log_message(self, *unused):+ pass++ with tempfile.TemporaryDirectory(prefix="headroom-cache-proof-") as temporary:+ log_path = args.log.resolve() if args.log else Path(temporary) / "proxy.log"+ server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), Upstream)+ worker = threading.Thread(target=server.serve_forever, daemon=True)+ worker.start()+ # The native CLI does not publish its port when binding :0. Reserve a+ # candidate briefly; readiness detects exit if another process wins it.+ with socket.socket() as reservation:+ reservation.bind(("127.0.0.1", 0))+ port = reservation.getsockname()[1]+ env = {+ key: value+ for key, value in os.environ.items()+ if not key.upper().startswith(+ (+ "HEADROOM_",+ "AWS_",+ "ANTHROPIC_",+ "OPENAI_",+ "GOOGLE_",+ "AZURE_",+ )+ )+ }+ env.update(+ AWS_EC2_METADATA_DISABLED="true",+ NO_PROXY="127.0.0.1,localhost",+ HEADROOM_OFFLINE="1",+ HF_HUB_OFFLINE="1",+ HEADROOM_LIVE_ZONE_DISABLE_ARMS="source_code,plain_text",+ )+ command = [+ str(binary),+ "--listen",+ f"127.0.0.1:{port}",+ "--upstream",+ f"http://127.0.0.1:{server.server_port}",+ "--compression",+ "--compression-mode",+ "off",+ "--stats",+ "false",+ "--log-level",+ "info",+ ]+ # Explicitly bypass caller HTTP proxy settings for all proof requests.+ client = urllib.request.build_opener(urllib.request.ProxyHandler({}))+ process = None+ try:+ with log_path.open("w", encoding="utf-8", newline="\n") as log:+ process = subprocess.Popen(+ command,+ env=env,+ cwd=temporary,+ stdout=log,+ stderr=subprocess.STDOUT,+ )+ deadline = time.monotonic() + 30+ while time.monotonic() < deadline:+ if process.poll() is not None:+ raise RuntimeError(f"proxy exited: {process.returncode}")+ try:+ with client.open(f"http://127.0.0.1:{port}/healthz", timeout=1) as response:+ response.read()+ break+ except OSError:+ time.sleep(0.2)+ else:+ raise RuntimeError("proxy readiness timeout")++ def request(system, token="", session="same"):+ payload = {+ "model": "claude-sonnet-4-20250514",+ "max_tokens": 16,+ "system": system,+ "messages": [{"role": "user", "content": "hello local"}],+ }+ headers = {"content-type": "application/json", "x-headroom-session-id": session}+ if token:+ headers["anthropic-beta"] = token+ outgoing = urllib.request.Request(+ f"http://127.0.0.1:{port}/v1/messages",+ data=json.dumps(payload).encode(),+ headers=headers,+ )+ with client.open(outgoing, timeout=30) as response:+ if (+ response.status != 200+ or json.load(response)["content"][0]["text"] != "local stub"+ ):+ raise RuntimeError("unexpected forwarded response")++ request("first system", "beta-a")+ request("changed system", "beta-b")+ request("changed system")+ if captured[1][1] != "beta-a,beta-b" or captured[2][1] != "beta-a,beta-b":+ raise RuntimeError(f"sequential beta union lost: {captured}")+ with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:+ list(+ pool.map(+ lambda i: request("parallel system", f"token-{i}", "parallel"), range(8)+ )+ )+ request("parallel system", session="parallel")+ if set(captured[-1][1].split(",")) != {f"token-{i}" for i in range(8)}:+ raise RuntimeError(f"concurrent beta union lost: {captured[-1]}")+ if len(captured) != 12:+ raise RuntimeError(f"expected 12 upstream requests, got {len(captured)}")+ log.flush()+ events = log_path.read_text(encoding="utf-8")+ if (+ "cache_drift_first_request" not in events+ or "cache_drift_observed" not in events+ ):+ raise RuntimeError("expected first-seen and changed-system drift events")+ print(+ json.dumps(+ {+ "passed": True,+ "upstream_requests": len(captured),+ "sequential_beta_union": captured[2][1],+ "parallel_beta_tokens": sorted(captured[-1][1].split(",")),+ "drift_first_seen_and_changed": True,+ }+ )+ )+ except Exception:+ if log_path.exists():+ print(log_path.read_text(encoding="utf-8"))+ raise+ finally:+ if process is not None:+ process.terminate()+ try:+ process.wait(timeout=10)+ except subprocess.TimeoutExpired:+ process.kill()+ process.wait()+ server.shutdown()+ server.server_close()+ worker.join(timeout=5)+++if __name__ == "__main__":+ main()