Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(client): close upstream responses when wrapped streams are closed (#4482)
Eren Şahin3d8dcf476c14
diff --git a/headroom/client.py b/headroom/client.pyindex 4d5ed50da..81b58cc63 100644--- a/headroom/client.py+++ b/headroom/client.py@@ -3,7 +3,7 @@from __future__ import annotationsimport logging-from collections.abc import Iterator+from collections.abc import Callable, Iterable, Iteratorfrom datetime import datetime, timezonefrom typing import Any@@ -38,6 +38,74 @@ from .utils import (logger = logging.getLogger(__name__)+class _OwnedStream(Iterator[Any]):+ """Iterator that owns and closes an upstream response stream."""++ def __init__(self, stream: Iterable[Any], save_metrics: Callable[[], None]) -> None:+ self._source: Iterable[Any] | None = stream+ self._stream: Iterator[Any] | None = None+ self._save_metrics: Callable[[], None] | None = save_metrics+ self._finished = False++ def __iter__(self) -> _OwnedStream:+ return self++ def __next__(self) -> Any:+ if self._finished:+ raise StopIteration+ try:+ if self._stream is None:+ source = self._source+ if source is None:+ raise StopIteration+ self._stream = iter(source)+ return next(self._stream)+ except BaseException:+ self.close()+ raise++ def close(self) -> None:+ """Close the upstream stream and save metrics exactly once."""+ if self._finished:+ return+ self._finished = True+ source = self._source+ stream = self._stream+ save_metrics = self._save_metrics+ try:+ candidates = (source,) if source is stream else (source, stream)+ for candidate in candidates:+ if candidate is None:+ continue+ close = getattr(candidate, "close", None)+ if close is not None:+ try:+ close()+ except Exception:+ logger.warning("Failed to close upstream response stream", exc_info=True)+ finally:+ self._source = None+ self._stream = None+ self._save_metrics = None+ if save_metrics is not None:+ save_metrics()++ def __enter__(self) -> _OwnedStream:+ return self++ def __exit__(self, exc_type: Any, exc_val: Any, exc_tb: Any) -> None:+ self.close()++ def __del__(self) -> None:+ # The previous generator wrapper ran its finally block when abandoned+ # after iteration started. Preserve that cleanup for callers that do+ # not explicitly close a stream.+ try:+ self.close()+ except Exception:+ logger.debug("Failed to finalize abandoned response stream", exc_info=True)++class ChatCompletions:"""Wrapper for chat.completions API (OpenAI-style)."""@@ -644,16 +712,11 @@ class HeadroomClient:def _wrap_stream(self,- stream: Iterator[Any],+ stream: Iterable[Any],metrics: RequestMetrics,) -> Iterator[Any]:- """Wrap stream to pass through chunks and save metrics at end."""- try:- yield from stream- finally:- # Save metrics when stream completes- # Note: output tokens unknown for streams- self._storage.save(metrics)+ """Wrap stream, closing the upstream and saving metrics exactly once."""+ return _OwnedStream(stream, lambda: self._storage.save(metrics))def _extract_query(self, messages: list[dict[str, Any]]) -> str:"""Extract query from messages for semantic caching.diff --git a/scripts/repro_client_stream_close.py b/scripts/repro_client_stream_close.pynew file mode 100644index 000000000..9f074a463--- /dev/null+++ b/scripts/repro_client_stream_close.py@@ -0,0 +1,372 @@+"""Reproduce client stream cleanup against a local SSE server.++Run the unread-stream proof with the checked-in defaults::++ uv run python scripts/repro_client_stream_close.py+ uv run python scripts/repro_client_stream_close.py --cycles 1001++Pass ``--started`` to consume the first event before closing each stream.+The baseline client method is loaded from ``--baseline-ref`` in a fresh child+process; it is never taken from the current checkout.+"""++from __future__ import annotations++import argparse+import ast+import ctypes+import gc+import json+import os+import subprocess+import sys+import tempfile+import threading+import time+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer+from pathlib import Path++BASELINE_REF = "3fef15d376d9899f77a28464e2e91f8b99514930"+++def rss_bytes() -> int | None:+ """Return this process's RSS using stdlib APIs where available."""+ if sys.platform == "win32":+ from ctypes import wintypes++ class Counters(ctypes.Structure):+ _fields_ = [("cb", wintypes.DWORD), ("PageFaultCount", wintypes.DWORD)] + [+ (name, ctypes.c_size_t)+ for name in (+ "PeakWorkingSetSize",+ "WorkingSetSize",+ "QuotaPeakPagedPoolUsage",+ "QuotaPagedPoolUsage",+ "QuotaPeakNonPagedPoolUsage",+ "QuotaNonPagedPoolUsage",+ "PagefileUsage",+ "PeakPagefileUsage",+ "PrivateUsage",+ )+ ]++ counters = Counters()+ counters.cb = ctypes.sizeof(counters)+ kernel = ctypes.WinDLL("kernel32", use_last_error=True)+ kernel.GetCurrentProcess.restype = wintypes.HANDLE+ psapi = ctypes.WinDLL("psapi", use_last_error=True)+ psapi.GetProcessMemoryInfo.argtypes = [+ wintypes.HANDLE,+ ctypes.POINTER(Counters),+ wintypes.DWORD,+ ]+ if psapi.GetProcessMemoryInfo(+ kernel.GetCurrentProcess(), ctypes.byref(counters), counters.cb+ ):+ return int(counters.WorkingSetSize)+ return None+ if sys.platform.startswith("linux"):+ try:+ resident_pages = int(Path("/proc/self/statm").read_text().split()[1])+ return resident_pages * os.sysconf("SC_PAGE_SIZE")+ except (OSError, IndexError, ValueError):+ pass+ return None+++def pool_connections(client: object) -> int | None:+ try:+ return len(client._client._transport._pool.connections) # type: ignore[attr-defined]+ except (AttributeError, TypeError):+ return None+++def load_baseline_method(module: object, ref: str) -> None:+ """Install exactly ``HeadroomClient._wrap_stream`` from ``git show``."""+ source = subprocess.check_output(["git", "show", f"{ref}:headroom/client.py"], text=True)+ tree = ast.parse(source)+ client_class = next(+ node+ for node in tree.body+ if isinstance(node, ast.ClassDef) and node.name == "HeadroomClient"+ )+ method = next(+ node+ for node in client_class.body+ if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node.name == "_wrap_stream"+ )+ namespace = dict(vars(module))+ code = compile(+ ast.fix_missing_locations(ast.Module(body=[method], type_ignores=[])), "<baseline>", "exec"+ )+ exec(code, namespace)+ module.HeadroomClient._wrap_stream = namespace["_wrap_stream"] # type: ignore[attr-defined]+++class SSEHandler(BaseHTTPRequestHandler):+ protocol_version = "HTTP/1.1"++ def log_message(self, *_args: object) -> None:+ pass++ def do_POST(self) -> None: # noqa: N802+ self.rfile.read(int(self.headers["Content-Length"]))+ self.send_response(200)+ self.send_header("Content-Type", "text/event-stream")+ self.send_header("Content-Length", "1000000")+ self.end_headers()+ self.wfile.write(+ b'data: {"id":"local","object":"chat.completion.chunk",'+ b'"created":0,"model":"local","choices":[{"index":0,'+ b'"delta":{"content":"hi"},"finish_reason":null}]}'+ b"\n\n"+ )+ self.wfile.flush()+ self.connection.settimeout(120)+ try:+ self.connection.recv(1)+ except OSError:+ pass+ self.close_connection = True+++def write_progress(path: Path, completed: int, samples: list[dict[str, object]]) -> None:+ path.write_text(+ json.dumps({"completed": completed, "samples": samples}),+ encoding="utf-8",+ newline="\n",+ )+++def child(args: argparse.Namespace) -> None:+ import httpx+ from openai import OpenAI++ import headroom.client as client_module+ from headroom.providers.openai import OpenAIProvider++ if args.variant == "baseline":+ load_baseline_method(client_module, args.baseline_ref)++ progress_path = Path(args.progress)+ samples: list[dict[str, object]] = []+ with OpenAI(+ api_key="local-only",+ base_url=args.url,+ max_retries=0,+ http_client=httpx.Client(+ trust_env=False,+ limits=httpx.Limits(max_connections=1000),+ timeout=httpx.Timeout(10, pool=0.25),+ ),+ ) as original:+ client = client_module.HeadroomClient(+ original_client=original,+ provider=OpenAIProvider(),+ store_url=f"jsonl://{Path(args.store).resolve().as_posix()}",+ enable_cache_optimizer=False,+ )++ def request() -> None:+ stream = client.chat.completions.create(+ model="gpt-4o-mini",+ messages=[{"role": "user", "content": "hi"}],+ stream=True,+ )+ if args.started:+ next(stream)+ stream.close()+ del stream++ for _ in range(5):+ warmup = original.chat.completions.create(+ model="gpt-4o-mini",+ messages=[{"role": "user", "content": "hi"}],+ stream=True,+ )+ next(warmup)+ warmup.close()+ del warmup+ warmup = client.chat.completions.create(+ model="gpt-4o-mini",+ messages=[{"role": "user", "content": "hi"}],+ stream=True,+ )+ next(warmup)+ warmup.close()+ del warmup+ gc.collect()+ import tracemalloc++ if not args.no_trace:+ tracemalloc.start(1)++ def sample(phase: str, cycles: int) -> None:+ heap, peak = tracemalloc.get_traced_memory()+ samples.append(+ {+ "phase": phase,+ "cycles": cycles,+ "rss": rss_bytes(),+ "heap": heap,+ "heap_peak": peak,+ "connections": pool_connections(original),+ }+ )+ write_progress(progress_path, completed, samples)++ completed = 0+ sample("baseline", 0)+ started_at = time.perf_counter()+ error: dict[str, object] | None = None+ for request_number in range(1, args.cycles + 1):+ try:+ request()+ except Exception as exc: # includes httpx.PoolTimeout+ cause = exc.__cause__+ error = {+ "type": type(exc).__name__,+ "cause": type(cause).__name__ if cause is not None else None,+ "request": request_number,+ "completed": completed,+ "pool_timeout": type(exc).__name__ == "PoolTimeout"+ or type(cause).__name__ == "PoolTimeout",+ }+ break+ completed += 1+ if request_number in (10, 50, 100, 250, 500, 1000, args.cycles):+ sample("running", request_number)+ elapsed = time.perf_counter() - started_at+ sample("before_gc", completed)+ gc.collect()+ time.sleep(0.05)+ sample("after_gc", completed)++ print(+ json.dumps(+ {+ "variant": args.variant,+ "started": args.started,+ "cycles": args.cycles,+ "completed": completed,+ "error": error,+ "elapsed": elapsed,+ "gc_enabled": gc.isenabled(),+ "samples": samples,+ }+ ),+ flush=True,+ )+++def stop_process(process: subprocess.Popen[str]) -> None:+ if os.name == "nt":+ subprocess.run(+ ["taskkill", "/PID", str(process.pid), "/T", "/F"],+ capture_output=True,+ check=False,+ )+ else:+ process.kill()+ try:+ process.communicate(timeout=5)+ except subprocess.TimeoutExpired:+ process.kill()+ process.communicate()+++def run_child(+ args: argparse.Namespace,+ server_port: int,+ variant: str,+ temporary: Path,+) -> dict[str, object]:+ progress = temporary / "progress.json"+ store = temporary / "metrics.jsonl"+ command = [+ sys.executable,+ str(Path(__file__).resolve()),+ "--child",+ "--variant",+ variant,+ "--baseline-ref",+ args.baseline_ref,+ "--cycles",+ str(args.cycles),+ "--url",+ f"http://127.0.0.1:{server_port}/v1",+ "--progress",+ str(progress),+ "--store",+ str(store),+ ]+ if args.started:+ command.append("--started")+ if args.no_trace:+ command.append("--no-trace")+ process = subprocess.Popen(+ command,+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ text=True,+ env={**os.environ, "HEADROOM_OFFLINE": "1"},+ )+ try:+ stdout, stderr = process.communicate(timeout=args.timeout)+ except subprocess.TimeoutExpired:+ stop_process(process)+ record: dict[str, object] = {+ "variant": variant,+ "started": args.started,+ "cycles": args.cycles,+ "completed": None,+ "error": {"type": "TimeoutExpired"},+ }+ if progress.exists():+ record.update(json.loads(progress.read_text(encoding="utf-8")))+ return record+ if process.returncode:+ raise RuntimeError(+ f"{variant} child exited {process.returncode}: {stdout[-1000:]} {stderr[-1000:]}"+ )+ return json.loads(stdout)+++def parent(args: argparse.Namespace) -> None:+ server = ThreadingHTTPServer(("127.0.0.1", 0), SSEHandler)+ server.daemon_threads = True+ threading.Thread(target=server.serve_forever, daemon=True).start()+ try:+ for repeat in range(1, args.repeats + 1):+ for variant in ("baseline", "fixed"):+ with tempfile.TemporaryDirectory(prefix="headroom-stream-close-") as directory:+ record = run_child(args, server.server_port, variant, Path(directory))+ record["repeat"] = repeat+ print(json.dumps(record), flush=True)+ finally:+ server.shutdown()+ server.server_close()+++def parse_args() -> argparse.Namespace:+ parser = argparse.ArgumentParser(description=__doc__)+ parser.add_argument("--child", action="store_true", help=argparse.SUPPRESS)+ parser.add_argument("--started", action="store_true")+ parser.add_argument("--cycles", type=int, default=500)+ parser.add_argument("--repeats", type=int, default=3)+ parser.add_argument("--baseline-ref", default=BASELINE_REF)+ parser.add_argument("--variant", choices=("baseline", "fixed"), help=argparse.SUPPRESS)+ parser.add_argument("--url", help=argparse.SUPPRESS)+ parser.add_argument("--progress", help=argparse.SUPPRESS)+ parser.add_argument("--store", help=argparse.SUPPRESS)+ parser.add_argument("--no-trace", action="store_true")+ parser.add_argument("--timeout", type=int, default=30)+ return parser.parse_args()+++if __name__ == "__main__":+ arguments = parse_args()+ if arguments.child:+ child(arguments)+ else:+ parent(arguments)diff --git a/tests/test_client_stream_lifecycle.py b/tests/test_client_stream_lifecycle.pynew file mode 100644index 000000000..4eb42ebc7--- /dev/null+++ b/tests/test_client_stream_lifecycle.py@@ -0,0 +1,316 @@+from __future__ import annotations++import gc+import weakref+from types import SimpleNamespace++import httpx+import pytest+from openai import OpenAI++from headroom.client import HeadroomClient+++class _ClosableIterator:+ def __init__(+ self,+ values: list[object] | None = None,+ error: Exception | None = None,+ close_error: Exception | None = None,+ ):+ self._values = iter(values or [])+ self._error = error+ self._close_error = close_error+ self.closed = 0++ def __iter__(self):+ return self++ def __next__(self):+ if self._error is not None:+ error, self._error = self._error, None+ raise error+ return next(self._values)++ def close(self):+ self.closed += 1+ if self._close_error is not None:+ raise self._close_error+++class _IterableOnly:+ def __init__(self, values: list[object]):+ self._values = values+ self.closed = 0++ def __iter__(self):+ return iter(self._values)++ def close(self):+ self.closed += 1+++class _RetainedIterable:+ def __init__(self):+ self.iterator = None+ self.closed = 0++ def __iter__(self):+ def generate():+ try:+ yield "chunk"+ yield "later"+ finally:+ self.closed += 1++ self.iterator = generate()+ return self.iterator+++class _DistinctClosableSource:+ def __init__(+ self, owner_error: Exception | None = None, iterator_error: Exception | None = None+ ):+ self.owner_error = owner_error+ self.iterator_error = iterator_error+ self.closed = 0+ self.iterator = None++ def __iter__(self):+ self.iterator = _ClosableIterator(["chunk"], close_error=self.iterator_error)+ return self.iterator++ def close(self):+ self.closed += 1+ if self.owner_error is not None:+ raise self.owner_error+++class _Owner:+ def __init__(self):+ self.saved = []+ self._storage = SimpleNamespace(save=self.saved.append)+++class _ClosableSse(httpx.SyncByteStream):+ def __init__(self):+ self.closed = 0++ def __iter__(self):+ yield (+ b'data: {"id":"x","object":"chat.completion.chunk","created":1,'+ b'"model":"gpt","choices":[{"index":0,"delta":{"content":"hi"},'+ b'"finish_reason":null}]}\n\n'+ )+ yield b"data: [DONE]\n\n"++ def close(self):+ self.closed += 1+++def _openai_stream(body: _ClosableSse):+ def handler(request: httpx.Request):+ return httpx.Response(+ 200,+ headers={"content-type": "text/event-stream"},+ stream=body,+ )++ http_client = httpx.Client(transport=httpx.MockTransport(handler))+ client = OpenAI(api_key="test", http_client=http_client)+ stream = client.chat.completions.create(+ model="gpt-test",+ messages=[{"role": "user", "content": "hello"}],+ stream=True,+ )+ return stream, http_client+++def test_wrap_stream_closes_source_before_iteration_and_saves_once():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ wrapped.close()+ wrapped.close()++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_closes_source_after_exhaustion_and_saves_once():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ assert list(wrapped) == ["chunk"]+ wrapped.close()++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_does_not_iterate_after_close():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ wrapped.close()++ with pytest.raises(StopIteration):+ next(wrapped)+ assert owner.saved == ["metrics"]+++def test_wrap_stream_closes_source_on_error_and_saves_once():+ owner = _Owner()+ source = _ClosableIterator(error=RuntimeError("boom"))+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ with pytest.raises(RuntimeError, match="boom"):+ next(wrapped)++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_supports_iterators_without_close():+ owner = _Owner()+ wrapped = HeadroomClient._wrap_stream(owner, iter(["chunk"]), "metrics")++ assert list(wrapped) == ["chunk"]+ assert owner.saved == ["metrics"]+++def test_wrap_stream_supports_iterable_only_sources():+ owner = _Owner()+ source = _IterableOnly(["chunk"])+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ assert list(wrapped) == ["chunk"]+ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_preserves_iteration_error_when_close_fails():+ owner = _Owner()+ source = _ClosableIterator(+ error=RuntimeError("iteration"),+ close_error=RuntimeError("close"),+ )+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ with pytest.raises(RuntimeError, match="iteration"):+ next(wrapped)++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_closes_retained_iterator_when_source_has_no_close():+ owner = _Owner()+ source = _RetainedIterable()+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ assert next(wrapped) == "chunk"+ wrapped.close()++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_closes_distinct_source_and_iterator():+ owner = _Owner()+ source = _DistinctClosableSource()+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ assert next(wrapped) == "chunk"+ wrapped.close()++ assert source.closed == 1+ assert source.iterator.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_closes_both_owners_when_each_close_fails():+ owner = _Owner()+ source = _DistinctClosableSource(+ owner_error=RuntimeError("owner close"),+ iterator_error=RuntimeError("iterator close"),+ )+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ assert next(wrapped) == "chunk"+ wrapped.close()+ wrapped.close()++ assert source.closed == 1+ assert source.iterator.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_stream_finalizes_started_abandoned_stream():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ next(wrapped)+ del wrapped+ gc.collect()++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_closed_stream_releases_source_and_owner():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])+ owner_ref, source_ref = weakref.ref(owner), weakref.ref(source)+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ wrapped.close()+ del owner, source+ gc.collect()++ assert owner_ref() is None+ assert source_ref() is None+++def test_wrap_stream_context_manager_closes_on_consumer_error():+ owner = _Owner()+ source = _ClosableIterator(["chunk"])++ with pytest.raises(RuntimeError, match="consumer"):+ with HeadroomClient._wrap_stream(owner, source, "metrics") as wrapped:+ assert next(wrapped) == "chunk"+ raise RuntimeError("consumer")++ assert source.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_real_openai_stream_closes_before_first_chunk():+ body = _ClosableSse()+ source, http_client = _openai_stream(body)+ owner = _Owner()+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ wrapped.close()+ http_client.close()++ assert body.closed == 1+ assert owner.saved == ["metrics"]+++def test_wrap_real_openai_stream_closes_after_first_chunk():+ body = _ClosableSse()+ source, http_client = _openai_stream(body)+ owner = _Owner()+ wrapped = HeadroomClient._wrap_stream(owner, source, "metrics")++ next(wrapped)+ wrapped.close()+ http_client.close()++ assert body.closed == 1+ assert owner.saved == ["metrics"]