Use Claude Code, Codex, VSCode, Pi, and OpenCode (and 6 other harnesses) for free (1.3B+ free tokens) from your terminal, app, IDE, or phone, and now from the browser with native browser sessions (multi-harness + multi-model) like OpenClaw (voice supported + ToS friendly)
Python
Request streamed usage for OpenAI-chat providers (#1013)
Ali Khokhardac6d4e88d4f
diff --git a/ARCHITECTURE.md b/ARCHITECTURE.mdindex 29f9f00b..51a6aa9e 100644--- a/ARCHITECTURE.md+++ b/ARCHITECTURE.md@@ -381,6 +381,11 @@ behavior matches shared transport policy. Provider-specific gateway quirks, suchas Cohere's supported `reasoning_effort` values, GitHub's API headers/catalogfiltering, Hugging Face's disabled prior reasoning replay, and unsupportedcompatibility fields, stay in that provider package.+The OpenAI-chat transport owns standard streamed usage handling: it requests+`stream_options.include_usage`, consumes provider `prompt_tokens` and+`completion_tokens` when present, and falls back to local estimates when+providers omit or reject optional usage metadata. Provider modules only own true+usage quirks such as DeepSeek prompt-cache counters.### Adding A Providerdiff --git a/core/anthropic/streaming/ledger.py b/core/anthropic/streaming/ledger.pyindex 241e3733..cae5f80f 100644--- a/core/anthropic/streaming/ledger.py+++ b/core/anthropic/streaming/ledger.py@@ -210,10 +210,13 @@ class AnthropicStreamLedger:stop_reason: str,output_tokens: int | None,*,+ input_tokens: int | None = None,usage_fields: Mapping[str, int] | None = None,) -> str:self.stop_reason = stop_reason- safe_in = _safe_usage_int(self.input_tokens)+ safe_in = _safe_usage_int(+ self.input_tokens if input_tokens is None else input_tokens+ )safe_out = output_tokens if isinstance(output_tokens, int) else 0usage = {"input_tokens": safe_in, "output_tokens": safe_out}if usage_fields:diff --git a/providers/deepseek/client.py b/providers/deepseek/client.pyindex b6fffc15..2c991176 100644--- a/providers/deepseek/client.py+++ b/providers/deepseek/client.py@@ -1,11 +1,11 @@"""DeepSeek provider implementation (OpenAI-compatible Chat Completions)."""-from collections.abc import Mappingfrom typing import Anyfrom providers.base import ProviderConfigfrom providers.defaults import DEEPSEEK_DEFAULT_BASEfrom providers.transports.openai_chat import OpenAIChatTransport+from providers.transports.openai_chat.usage import usage_intfrom .compat import build_deepseek_request_body@@ -31,24 +31,10 @@ class DeepSeekProvider(OpenAIChatTransport):def _anthropic_usage_fields(self, usage_info: Any) -> dict[str, int]:usage_fields: dict[str, int] = {}- cache_hit_tokens = _usage_int(usage_info, "prompt_cache_hit_tokens")+ cache_hit_tokens = usage_int(usage_info, "prompt_cache_hit_tokens")if cache_hit_tokens is not None:usage_fields["cache_read_input_tokens"] = cache_hit_tokens- cache_miss_tokens = _usage_int(usage_info, "prompt_cache_miss_tokens")+ cache_miss_tokens = usage_int(usage_info, "prompt_cache_miss_tokens")if cache_miss_tokens is not None:usage_fields["cache_creation_input_tokens"] = cache_miss_tokensreturn usage_fields---def _usage_int(usage_info: Any, key: str) -> int | None:- if usage_info is None:- return None- if isinstance(usage_info, Mapping):- value = usage_info.get(key)- else:- value = getattr(usage_info, key, None)- if value is None:- extra = getattr(usage_info, "model_extra", None)- if isinstance(extra, Mapping):- value = extra.get(key)- return value if isinstance(value, int) else Nonediff --git a/providers/deepseek/compat.py b/providers/deepseek/compat.pyindex 5448744b..f755332f 100644--- a/providers/deepseek/compat.py+++ b/providers/deepseek/compat.py@@ -428,10 +428,6 @@ def _request_from_dict(request_data: Any, data: dict[str, Any]) -> Any:def _apply_deepseek_chat_extras(body: dict[str, Any], _request_data: Any, thinking_enabled: bool) -> None:- stream_options = body.setdefault("stream_options", {})- if isinstance(stream_options, dict):- stream_options["include_usage"] = True-if not thinking_enabled or body.get("model") == "deepseek-reasoner":returnextra_body = body.setdefault("extra_body", {})diff --git a/providers/transports/openai_chat/stream.py b/providers/transports/openai_chat/stream.pyindex be719e63..e24de9fd 100644--- a/providers/transports/openai_chat/stream.py+++ b/providers/transports/openai_chat/stream.py@@ -31,6 +31,7 @@ from .tool_calls import (iter_heuristic_tool_use_sse,tool_call_extra_content,)+from .usage import request_stream_usage, usage_intclass OpenAIChatStreamAdapter:@@ -79,6 +80,7 @@ class OpenAIChatStreamAdapter:body = self._transport._build_request_body(self._request, thinking_enabled=self._thinking_enabled)+ request_stream_usage(body)thinking_enabled = self._transport._is_thinking_enabled(self._request, self._thinking_enabled)@@ -113,8 +115,9 @@ class OpenAIChatStreamAdapter:stream_opened = Truetool_argument_aliases = self._transport._tool_argument_aliases(body)async for chunk in stream:- if getattr(chunk, "usage", None):- usage_info = chunk.usage+ chunk_usage = getattr(chunk, "usage", None)+ if chunk_usage is not None:+ usage_info = chunk_usageif not chunk.choices:continue@@ -350,24 +353,22 @@ class OpenAIChatStreamAdapter:for event in hold_events(ledger.close_all_blocks()):yield event- completion = (- getattr(usage_info, "completion_tokens", None)- if usage_info is not None- else None- )+ completion = usage_int(usage_info, "completion_tokens")if isinstance(completion, int):output_tokens = completionelse:output_tokens = ledger.estimate_output_tokens()- if usage_info and hasattr(usage_info, "prompt_tokens"):- provider_input = usage_info.prompt_tokens- if isinstance(provider_input, int):- logger.debug(- "TOKEN_ESTIMATE: our={} provider={} diff={:+d}",- self._input_tokens,- provider_input,- provider_input - self._input_tokens,- )+ provider_input = usage_int(usage_info, "prompt_tokens")+ if provider_input is not None:+ logger.debug(+ "TOKEN_ESTIMATE: our={} provider={} diff={:+d}",+ self._input_tokens,+ provider_input,+ provider_input - self._input_tokens,+ )+ input_tokens = (+ provider_input if provider_input is not None else self._input_tokens+ )trace_event(stage="provider",event="provider.response.completed",@@ -375,12 +376,14 @@ class OpenAIChatStreamAdapter:provider=tag,finish_reason=(None if finish_reason is None else str(finish_reason)),output_tokens=output_tokens,+ prompt_tokens=input_tokens,prompt_tokens_estimate=self._input_tokens,)for event in hold_event(ledger.message_delta(ledger.final_stop_reason(map_stop_reason(finish_reason)),output_tokens,+ input_tokens=input_tokens,usage_fields=self._transport._anthropic_usage_fields(usage_info),)):diff --git a/providers/transports/openai_chat/transport.py b/providers/transports/openai_chat/transport.pyindex e3726c17..ae95fcfe 100644--- a/providers/transports/openai_chat/transport.py+++ b/providers/transports/openai_chat/transport.py@@ -20,6 +20,7 @@ from providers.rate_limit import GlobalRateLimiterfrom .output_cap import clamp_output_tokens, parse_output_token_capfrom .stream import OpenAIChatStreamAdapter+from .usage import clone_without_stream_usage, is_stream_usage_rejectionclass OpenAIChatTransport(BaseProvider):@@ -117,26 +118,51 @@ class OpenAIChatTransport(BaseProvider):return {}async def _create_stream(self, body: dict) -> tuple[Any, dict]:- """Create a streaming chat completion, optionally retrying once."""+ """Create a streaming chat completion with bounded request fallbacks."""body = self._apply_learned_output_cap(body)- try:- create_body = self._prepare_create_body(body)- stream = await self._global_rate_limiter.execute_with_retry(- self._client.chat.completions.create, **create_body, stream=True- )- return stream, body- except Exception as error:- retry_body = self._retry_body_for_output_cap(error, body)- if retry_body is None:- retry_body = self._get_retry_request_body(error, body)- if retry_body is None:- raise-- create_retry_body = self._prepare_create_body(retry_body)- stream = await self._global_rate_limiter.execute_with_retry(- self._client.chat.completions.create, **create_retry_body, stream=True- )- return stream, retry_body+ used_retry_kinds: set[str] = set()++ while True:+ try:+ create_body = self._prepare_create_body(body)+ stream = await self._global_rate_limiter.execute_with_retry(+ self._client.chat.completions.create, **create_body, stream=True+ )+ return stream, body+ except Exception as error:+ retry_body = self._next_create_retry_body(error, body, used_retry_kinds)+ if retry_body is None:+ raise+ body = retry_body++ def _next_create_retry_body(+ self,+ error: Exception,+ body: dict,+ used_retry_kinds: set[str],+ ) -> dict | None:+ retry_body = self._retry_body_for_output_cap(error, body)+ if retry_body is not None:+ return retry_body++ if "stream_usage" not in used_retry_kinds and is_stream_usage_rejection(error):+ retry_body = clone_without_stream_usage(body)+ if retry_body is not None:+ used_retry_kinds.add("stream_usage")+ logger.warning(+ "{}_STREAM: retrying without stream_options.include_usage "+ "after upstream rejection",+ self._provider_name,+ )+ return retry_body++ if "provider_specific" not in used_retry_kinds:+ retry_body = self._get_retry_request_body(error, body)+ if retry_body is not None:+ used_retry_kinds.add("provider_specific")+ return retry_body++ return Nonedef _apply_learned_output_cap(self, body: dict) -> dict:"""Clamp output tokens to a previously learned cap for this model."""diff --git a/providers/transports/openai_chat/usage.py b/providers/transports/openai_chat/usage.pynew file mode 100644index 00000000..4be8dff1--- /dev/null+++ b/providers/transports/openai_chat/usage.py@@ -0,0 +1,98 @@+"""OpenAI-chat streamed usage request and extraction helpers."""++import json+from collections.abc import Mapping+from typing import Any++import openai++_USAGE_OPTION_KEYS = ("stream_options", "include_usage")+_USAGE_REJECTION_WORDS = (+ "unsupported",+ "not supported",+ "unknown",+ "unrecognized",+ "unexpected",+ "invalid",+ "extra",+ "forbidden",+ "not permitted",+)+++def request_stream_usage(body: dict[str, Any]) -> None:+ """Ask an OpenAI-compatible streaming endpoint for its final usage chunk."""+ stream_options = body.get("stream_options")+ if stream_options is None:+ body["stream_options"] = {"include_usage": True}+ return+ if isinstance(stream_options, dict):+ stream_options["include_usage"] = True+++def clone_without_stream_usage(body: dict[str, Any]) -> dict[str, Any] | None:+ """Return a clone with only ``include_usage`` removed from stream options."""+ stream_options = body.get("stream_options")+ if not isinstance(stream_options, dict):+ return None+ if "include_usage" not in stream_options:+ return None++ retry_body = dict(body)+ retry_stream_options = dict(stream_options)+ retry_stream_options.pop("include_usage", None)+ if retry_stream_options:+ retry_body["stream_options"] = retry_stream_options+ else:+ retry_body.pop("stream_options", None)+ return retry_body+++def is_stream_usage_rejection(error: Exception) -> bool:+ """Return whether upstream rejected the optional streamed-usage request."""+ if not _is_bad_request_like(error):+ return False+ text = _error_text(error)+ if not any(key in text for key in _USAGE_OPTION_KEYS):+ return False+ return any(word in text for word in _USAGE_REJECTION_WORDS)+++def usage_int(usage_info: Any, key: str) -> int | None:+ """Extract an integer usage field from OpenAI SDK objects or plain dicts."""+ if usage_info is None:+ return None+ if isinstance(usage_info, Mapping):+ value = usage_info.get(key)+ else:+ value = getattr(usage_info, key, None)+ if value is None:+ extra = getattr(usage_info, "model_extra", None)+ if isinstance(extra, Mapping):+ value = extra.get(key)+ return value if isinstance(value, int) and not isinstance(value, bool) else None+++def _is_bad_request_like(error: Exception) -> bool:+ if isinstance(error, openai.BadRequestError):+ return True+ status = getattr(error, "status_code", None)+ if status is None:+ response = getattr(error, "response", None)+ status = (+ getattr(response, "status_code", None) if response is not None else None+ )+ return status in (400, 422)+++def _error_text(error: Exception) -> str:+ parts = [str(error)]+ body = getattr(error, "body", None)+ if body is not None:+ parts.append(json.dumps(body, default=str))+ response = getattr(error, "response", None)+ if response is not None:+ text = getattr(response, "text", None)+ if isinstance(text, str) and text:+ parts.append(text)+ return " ".join(parts).lower()diff --git a/pyproject.toml b/pyproject.tomlindex c36c375b..829bceee 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -4,7 +4,7 @@ build-backend = "hatchling.build"[project]name = "free-claude-code"-version = "3.4.8"+version = "3.4.9"description = "Middleware between Claude Code CLI (Anthropic API) and NVIDIA NIM"readme = "README.md"requires-python = ">=3.14.0"diff --git a/tests/providers/test_deepseek.py b/tests/providers/test_deepseek.pyindex 7a976861..fd560c87 100644--- a/tests/providers/test_deepseek.py+++ b/tests/providers/test_deepseek.py@@ -82,7 +82,7 @@ def test_build_request_body_openai_chat_shape(deepseek_provider):assert body["messages"][1]["role"] == "user"assert body["messages"][1] == {"role": "user", "content": "Hello"}assert body["max_tokens"] == 100- assert body["stream_options"] == {"include_usage": True}+ assert "stream_options" not in bodydef test_build_request_body_default_max_tokens(deepseek_provider):@@ -188,7 +188,7 @@ def test_build_request_body_respects_global_thinking_disable():)body = provider._build_request_body(request)assert "extra_body" not in body- assert body["stream_options"] == {"include_usage": True}+ assert "stream_options" not in bodydef test_preserve_unsigned_thinking_when_thinking_on(deepseek_provider):@@ -714,7 +714,7 @@ async def test_stream_uses_chat_completions_and_maps_cache_usage(deepseek_providevent.data["usage"] for event in parsed if event.event == "message_delta")assert usage == {- "input_tokens": 7,+ "input_tokens": 30,"output_tokens": 3,"cache_read_input_tokens": 10,"cache_creation_input_tokens": 20,diff --git a/tests/providers/test_openai_chat_usage.py b/tests/providers/test_openai_chat_usage.pynew file mode 100644index 00000000..d07995e8--- /dev/null+++ b/tests/providers/test_openai_chat_usage.py@@ -0,0 +1,218 @@+"""OpenAI-chat streamed usage helper tests."""++from types import SimpleNamespace+from typing import Any+from unittest.mock import AsyncMock, patch++import openai+import pytest+from httpx import Request, Response++from core.anthropic.stream_contracts import parse_sse_text+from providers.base import ProviderConfig+from providers.rate_limit import GlobalRateLimiter+from providers.transports.openai_chat import OpenAIChatTransport+from providers.transports.openai_chat.usage import (+ clone_without_stream_usage,+ is_stream_usage_rejection,+ request_stream_usage,+ usage_int,+)+++class _UsageTestProvider(OpenAIChatTransport):+ def __init__(self):+ super().__init__(+ ProviderConfig(+ api_key="test_key",+ base_url="https://provider.example/v1",+ rate_limit=100,+ rate_window=60,+ ),+ provider_name="USAGE_TEST",+ base_url="https://provider.example/v1",+ api_key="test_key",+ )++ def _build_request_body(+ self, request: Any, thinking_enabled: bool | None = None+ ) -> dict:+ return {"model": request.model, "messages": [{"role": "user", "content": "x"}]}+++def _bad_request(message: str, body: object | None = None) -> openai.BadRequestError:+ response = Response(+ 400,+ request=Request("POST", "https://provider.example/v1/chat/completions"),+ )+ return openai.BadRequestError(message, response=response, body=body)+++async def _stream(chunks):+ for chunk in chunks:+ yield chunk+++def _chunk(+ *,+ content: str | None = None,+ finish_reason: str | None = None,+ usage: Any = None,+):+ if content is None and finish_reason is None:+ return SimpleNamespace(choices=[], usage=usage)+ return SimpleNamespace(+ choices=[+ SimpleNamespace(+ delta=SimpleNamespace(+ content=content,+ reasoning_content=None,+ tool_calls=None,+ ),+ finish_reason=finish_reason,+ )+ ],+ usage=usage,+ )+++def test_request_stream_usage_adds_stream_options_when_absent():+ body = {"model": "m"}++ request_stream_usage(body)++ assert body["stream_options"] == {"include_usage": True}+++def test_request_stream_usage_preserves_existing_stream_options():+ stream_options = {"foo": "bar"}+ body = {"model": "m", "stream_options": stream_options}++ request_stream_usage(body)++ assert body["stream_options"] == {"foo": "bar", "include_usage": True}+ assert body["stream_options"] is stream_options+++def test_clone_without_stream_usage_removes_only_include_usage():+ body = {+ "model": "m",+ "stream_options": {"foo": "bar", "include_usage": True},+ }++ retry_body = clone_without_stream_usage(body)++ assert retry_body == {"model": "m", "stream_options": {"foo": "bar"}}+ assert body["stream_options"] == {"foo": "bar", "include_usage": True}+++def test_clone_without_stream_usage_drops_empty_stream_options():+ body = {"model": "m", "stream_options": {"include_usage": True}}++ retry_body = clone_without_stream_usage(body)++ assert retry_body == {"model": "m"}+++def test_usage_int_reads_dict_object_and_model_extra():+ assert usage_int({"prompt_tokens": 11}, "prompt_tokens") == 11+ assert usage_int(SimpleNamespace(completion_tokens=7), "completion_tokens") == 7+ assert (+ usage_int(+ SimpleNamespace(model_extra={"prompt_cache_hit_tokens": 3}),+ "prompt_cache_hit_tokens",+ )+ == 3+ )+ assert usage_int(SimpleNamespace(prompt_tokens=None), "prompt_tokens") is None+ assert usage_int({"prompt_tokens": True}, "prompt_tokens") is None+++def test_stream_usage_rejection_matches_usage_option_400():+ error = _bad_request(+ "Unrecognized request argument supplied: stream_options",+ {"error": {"message": "stream_options is unsupported"}},+ )++ assert is_stream_usage_rejection(error)+++def test_stream_usage_rejection_does_not_match_unrelated_400():+ error = _bad_request(+ "messages: invalid role",+ {"error": {"message": "messages contains invalid role"}},+ )++ assert not is_stream_usage_rejection(error)+++async def test_openai_chat_stream_requests_usage_and_uses_provider_prompt_tokens():+ GlobalRateLimiter.reset_instance()+ try:+ provider = _UsageTestProvider()+ request = SimpleNamespace(model="m")+ usage = SimpleNamespace(prompt_tokens=22, completion_tokens=4)+ create = AsyncMock(+ return_value=_stream(+ [+ _chunk(content="hello"),+ _chunk(finish_reason="stop"),+ _chunk(usage=usage),+ ]+ )+ )++ with patch.object(provider._client.chat.completions, "create", create):+ events = [+ event+ async for event in provider.stream_response(request, input_tokens=7)+ ]++ create.assert_awaited_once()+ await_args = create.await_args+ assert await_args is not None+ assert await_args.kwargs["stream_options"] == {"include_usage": True}+ parsed = parse_sse_text("".join(events))+ start_usage = next(+ event.data["message"]["usage"]+ for event in parsed+ if event.event == "message_start"+ )+ final_usage = next(+ event.data["usage"] for event in parsed if event.event == "message_delta"+ )+ assert start_usage["input_tokens"] == 7+ assert final_usage == {"input_tokens": 22, "output_tokens": 4}+ finally:+ GlobalRateLimiter.reset_instance()+++async def test_openai_chat_stream_retries_without_usage_when_option_is_rejected():+ GlobalRateLimiter.reset_instance()+ try:+ provider = _UsageTestProvider()+ body = {"model": "m", "messages": [{"role": "user", "content": "x"}]}+ request_stream_usage(body)+ create = AsyncMock(+ side_effect=[+ _bad_request(+ "stream_options is unsupported",+ {"error": {"message": "stream_options is unsupported"}},+ ),+ object(),+ ]+ )++ with patch.object(provider._client.chat.completions, "create", create):+ _stream_obj, used_body = await provider._create_stream(body)++ assert create.await_count == 2+ assert create.await_args_list[0].kwargs["stream_options"] == {+ "include_usage": True+ }+ assert "stream_options" not in create.await_args_list[1].kwargs+ assert "stream_options" not in used_body+ finally:+ GlobalRateLimiter.reset_instance()diff --git a/uv.lock b/uv.lockindex 192d4b36..adc31cef 100644--- a/uv.lock+++ b/uv.lock@@ -561,7 +561,7 @@ wheels = [[[package]]name = "free-claude-code"-version = "3.4.8"+version = "3.4.9"source = { editable = "." }dependencies = [{ name = "aiohttp" },