Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(providers): request real token usage on streamed calls

Yufeng He84474865e523
diff --git a/agent/src/providers/llm.py b/agent/src/providers/llm.py
index 1fb5f2e1c..6fa71cfb5 100644
--- a/agent/src/providers/llm.py
+++ b/agent/src/providers/llm.py
@@ -530,7 +530,28 @@ if ChatOpenAI is not None:
cloned = copy(self)
cloned.root_client = _ResponsesSyncClient(self.root_client)
return super(ChatOpenAIWithReasoning, cloned)._stream(*args, **kwargs)
- return super()._stream(*args, **kwargs)
+ return self._stream_with_usage_fallback(*args, **kwargs)
+
+ def _stream_with_usage_fallback(self, *args: Any, **kwargs: Any) -> Iterator[Any]:
+ # stream_usage=True is set at build time so streamed calls report
+ # real token counts (issue #1224); an endpoint that rejects
+ # stream_options is remembered and retried without it.
+ model = str(self.model_name)
+ if model in _STREAM_USAGE_UNSUPPORTED:
+ yield from super()._stream(*args, stream_usage=False, **kwargs)
+ return
+ try:
+ yield from super()._stream(*args, **kwargs)
+ return
+ except Exception as exc: # noqa: BLE001 - classified below
+ if not _is_stream_usage_unsupported_error(exc):
+ raise
+ logger.warning(
+ "Endpoint rejects stream usage; streaming without stream_options for %s",
+ model,
+ )
+ _STREAM_USAGE_UNSUPPORTED.add(model)
+ yield from super()._stream(*args, stream_usage=False, **kwargs)
async def _astream(self, *args: Any, **kwargs: Any) -> AsyncIterator[Any]:
"""Async route matching ``_stream`` for Responses compatibility."""
@@ -541,9 +562,26 @@ if ChatOpenAI is not None:
*args, **kwargs
):
yield chunk
- else:
+ return
+ model = str(self.model_name)
+ if model in _STREAM_USAGE_UNSUPPORTED:
+ async for chunk in super()._astream(*args, stream_usage=False, **kwargs):
+ yield chunk
+ return
+ try:
async for chunk in super()._astream(*args, **kwargs):
yield chunk
+ return
+ except Exception as exc: # noqa: BLE001 - classified below
+ if not _is_stream_usage_unsupported_error(exc):
+ raise
+ logger.warning(
+ "Endpoint rejects stream usage; streaming without stream_options for %s",
+ model,
+ )
+ _STREAM_USAGE_UNSUPPORTED.add(model)
+ async for chunk in super()._astream(*args, stream_usage=False, **kwargs):
+ yield chunk
def _convert_chunk_to_generation_chunk( # type: ignore[override]
self,
@@ -816,6 +854,33 @@ def _native_deepseek_adapter_available() -> bool:
# process-wide to skip the redundant failed request on subsequent calls.
_ANTHROPIC_TEMPERATURE_UNSUPPORTED: set[str] = set()
+# Endpoints discovered at runtime to reject the `stream_options` request
+# field. OpenAI-compatible gateways vary on include_usage support; membership
+# is populated on first failure and reused process-wide to skip the redundant
+# failed request on later calls (issue #1224).
+_STREAM_USAGE_UNSUPPORTED: set[str] = set()
+
+
+def _is_stream_usage_unsupported_error(exc: BaseException) -> bool:
+ """Return True when an endpoint rejects the `stream_options` field.
+
+ Matches request-validation rejections of `stream_options`/`include_usage`
+ regardless of the SDK exception type.
+ """
+ message = str(getattr(exc, "message", "") or exc).lower()
+ if "stream_options" not in message and "include_usage" not in message:
+ return False
+ return (
+ "unsupported" in message
+ or "not supported" in message
+ or "unknown" in message
+ or "unrecognized" in message
+ or "invalid" in message
+ or "not valid" in message
+ or "not a valid" in message
+ or "not allowed" in message
+ )
+
# Cache of base ChatAnthropic class -> temperature-safe subclass, so the dynamic
# subclass is built once per resolved base class (keyed to support test doubles).
_TEMPERATURE_SAFE_ANTHROPIC_CACHE: dict[type, type] = {}
@@ -1422,6 +1487,9 @@ def build_llm(*, model_name: Optional[str] = None, callbacks: Any = None) -> Any
"timeout": get_env_config().llm.timeout_seconds,
"max_retries": get_env_config().llm.max_retries,
"callbacks": callbacks,
+ # Ask for real token usage on streamed calls (issue #1224); endpoints
+ # that reject stream_options self-heal in _stream_with_usage_fallback.
+ "stream_usage": True,
"output_version": "responses/v1" if use_responses_api else None,
"use_responses_api": use_responses_api,
"reasoning": (
diff --git a/agent/tests/test_provider_header_isolation.py b/agent/tests/test_provider_header_isolation.py
index cd07ee212..adb108f55 100644
--- a/agent/tests/test_provider_header_isolation.py
+++ b/agent/tests/test_provider_header_isolation.py
@@ -240,3 +240,127 @@ def test_provider_doctor_reports_header_safety_without_values() -> None:
encoded = json.dumps(diagnostics, ensure_ascii=False)
assert "sk-or-secret-value" not in encoded
assert "private-à" not in encoded
+
+
+def _sse_usage_body(text: str, output_tokens: int) -> bytes:
+ """SSE payload whose final chunk carries real usage (stream include_usage)."""
+ text_chunk = {
+ "id": "chatcmpl-usage-test",
+ "object": "chat.completion.chunk",
+ "created": 0,
+ "model": "gpt-usage-test",
+ "choices": [{"index": 0, "delta": {"content": text}, "finish_reason": None}],
+ }
+ usage_chunk = {
+ "id": "chatcmpl-usage-test",
+ "object": "chat.completion.chunk",
+ "created": 0,
+ "model": "gpt-usage-test",
+ "choices": [],
+ "usage": {
+ "prompt_tokens": 28,
+ "completion_tokens": output_tokens,
+ "total_tokens": 28 + output_tokens,
+ },
+ }
+ return (f"data: {json.dumps(text_chunk)}\n\ndata: {json.dumps(usage_chunk)}\n\ndata: [DONE]\n\n").encode("utf-8")
+
+
+ ChatOpenAIWithReasoning is None,
+ reason="langchain-openai is not installed",
+)
+def test_stream_requests_usage_and_forwards_real_counts() -> None:
+ """stream_usage=True must put stream_options on the wire and let the
+ accumulated response carry the provider's real token counts (#1224)."""
+ seen: dict[str, object] = {}
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ seen["body"] = request.content.decode("utf-8")
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ content=_sse_usage_body("ok", 241),
+ )
+
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ llm = ChatOpenAIWithReasoning(
+ model="gpt-usage-test",
+ api_key="sk-test",
+ base_url="https://api.openai.invalid/v1",
+ stream_usage=True,
+ http_client=client,
+ vibe_provider="openai",
+ vibe_api_key="sk-test",
+ )
+ accumulated = None
+ for chunk in llm.stream("hello"):
+ accumulated = chunk if accumulated is None else accumulated + chunk
+
+ assert '"stream_options":{"include_usage":true}' in str(seen["body"])
+ usage = getattr(accumulated, "usage_metadata", None)
+ assert usage is not None
+ assert usage["input_tokens"] == 28
+ assert usage["output_tokens"] == 241
+
+
+ ChatOpenAIWithReasoning is None,
+ reason="langchain-openai is not installed",
+)
+def test_stream_usage_rejection_self_heals_and_is_remembered() -> None:
+ """An endpoint that 400s on stream_options gets one stateless retry, and
+ later calls skip the doomed attempt entirely."""
+ import src.providers.llm as llm_mod
+
+ calls: list[str] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ body = request.content.decode("utf-8")
+ calls.append(body)
+ if '"stream_options"' in body:
+ return httpx.Response(
+ 400,
+ json={"error": {"message": "Unknown parameter: 'stream_options' is unsupported"}},
+ )
+ return httpx.Response(
+ 200,
+ headers={"content-type": "text/event-stream"},
+ content=_sse_usage_body("ok", 3),
+ )
+
+ model = "gpt-usage-reject-test"
+ llm_mod._STREAM_USAGE_UNSUPPORTED.discard(model)
+ with httpx.Client(transport=httpx.MockTransport(handler)) as client:
+ llm = ChatOpenAIWithReasoning(
+ model=model,
+ api_key="sk-test",
+ base_url="https://api.openai.invalid/v1",
+ stream_usage=True,
+ http_client=client,
+ vibe_provider="openai",
+ vibe_api_key="sk-test",
+ )
+ first = "".join(chunk.content for chunk in llm.stream("hello"))
+ second = "".join(chunk.content for chunk in llm.stream("hello again"))
+
+ assert first == "ok"
+ assert second == "ok"
+ # First call: one rejected attempt plus the stateless retry. Second call:
+ # straight to no-usage, no wasted 400.
+ assert len(calls) == 3
+ assert '"stream_options"' in calls[0]
+ assert '"stream_options"' not in calls[1]
+ assert '"stream_options"' not in calls[2]
+ llm_mod._STREAM_USAGE_UNSUPPORTED.discard(model)
+
+
+def test_stream_usage_unsupported_error_detection() -> None:
+ import src.providers.llm as llm_mod
+
+ assert llm_mod._is_stream_usage_unsupported_error(ValueError("Unknown parameter: 'stream_options' is unsupported"))
+ assert llm_mod._is_stream_usage_unsupported_error(
+ ValueError("include_usage is not a valid field for this endpoint")
+ )
+ assert not llm_mod._is_stream_usage_unsupported_error(ValueError("model overloaded, retry later"))
+ assert not llm_mod._is_stream_usage_unsupported_error(ValueError("temperature is unsupported"))