The agent engineering platform.
Python agentsaiai-agentsanthropicchatgptdeepagentsenterpriseframeworkgeminigenerative-ailangchainlanggraphllmmultiagentopen-sourceopenaipydanticpythonragtypescript
fix(fireworks): classify mid-stream read timeouts (#40874)
Mason Daugherty60e57f5bc5e4
diff --git a/libs/partners/fireworks/langchain_fireworks/chat_models.py b/libs/partners/fireworks/langchain_fireworks/chat_models.pyindex 3b10e157b9..ff38f7e3a7 100644--- a/libs/partners/fireworks/langchain_fireworks/chat_models.py+++ b/libs/partners/fireworks/langchain_fireworks/chat_models.py@@ -653,6 +653,18 @@ class FireworksTimeoutError(APITimeoutError, ModelTimeoutError):"""Fireworks timeout error classified as a LangChain model error."""+class FireworksReadTimeoutError(httpx.ReadTimeout, ModelTimeoutError):+ """Fireworks stream read timeout classified as a retryable model error."""+++def _handle_stream_read_timeout(error: httpx.ReadTimeout) -> NoReturn:+ try:+ request = error.request+ except RuntimeError:+ request = None+ raise FireworksReadTimeoutError(str(error), request=request) from error++def _handle_fireworks_invalid_request(e: BadRequestError) -> NoReturn:"""Promote prompt-too-long errors to `FireworksContextOverflowError`."""if "prompt is too long" in str(e):@@ -815,13 +827,19 @@ async def _acompletion_with_retry(def _prepend_chunk(first: Any, rest: Iterator[Any]) -> Iterator[Any]:yield first- yield from rest+ try:+ yield from rest+ except httpx.ReadTimeout as e:+ _handle_stream_read_timeout(e)async def _aprepend_chunk(first: Any, rest: AsyncIterator[Any]) -> AsyncIterator[Any]:yield first- async for item in rest:- yield item+ try:+ async for item in rest:+ yield item+ except httpx.ReadTimeout as e:+ _handle_stream_read_timeout(e)class ChatFireworks(BaseChatModel):diff --git a/libs/partners/fireworks/tests/unit_tests/test_chat_models.py b/libs/partners/fireworks/tests/unit_tests/test_chat_models.pyindex 2679d550b2..5ac6fefc26 100644--- a/libs/partners/fireworks/tests/unit_tests/test_chat_models.py+++ b/libs/partners/fireworks/tests/unit_tests/test_chat_models.py@@ -5,8 +5,9 @@ from __future__ import annotationsimport jsonimport loggingimport os+from collections.abc import AsyncIterator, Iteratorfrom typing import Any-from unittest.mock import MagicMock+from unittest.mock import AsyncMock, MagicMockimport httpximport pytest@@ -841,7 +842,11 @@ def test_completion_with_retry_exhausts_and_raises() -> None:assert mock_client.create.call_count == 3-def test_completion_with_retry_streaming_retries_on_setup() -> None:+ "error",+ [_api_error(RateLimitError, "rate limited", 429), httpx.ReadTimeout("slow")],+)+def test_completion_with_retry_streaming_retries_on_setup(error: Exception) -> None:"""Streaming errors raised during the first-chunk pull are retried."""llm = _make_llm(max_retries=1)@@ -852,8 +857,7 @@ def test_completion_with_retry_streaming_retries_on_setup() -> None:if calls["n"] == 1:def _failing_gen() -> Any:- msg = "rate limited"- raise _api_error(RateLimitError, msg, 429)+ raise erroryield # pragma: no coverreturn _failing_gen()@@ -994,7 +998,13 @@ def test_chat_fireworks_invoke_routes_through_retry() -> None:assert mock_client.create.call_count == 2-async def test_acompletion_with_retry_streaming_retries_on_setup() -> None:+ "error",+ [_api_error(RateLimitError, "rate limited", 429), httpx.ReadTimeout("slow")],+)+async def test_acompletion_with_retry_streaming_retries_on_setup(+ error: Exception,+) -> None:"""Async streaming errors during the first-chunk pull are retried."""llm = _make_llm(max_retries=1)calls = {"n": 0}@@ -1004,8 +1014,7 @@ async def test_acompletion_with_retry_streaming_retries_on_setup() -> None:if calls["n"] == 1:async def _failing_agen() -> Any:- msg = "rate limited"- raise _api_error(RateLimitError, msg, 429)+ raise erroryield # pragma: no coverreturn _failing_agen()@@ -1546,6 +1555,96 @@ class TestExtraHeaders:assert "extra_headers" not in call_kwargs.get("extra_body", {})[email protected]("with_request", [False, True])[email protected]("async_mode", [False, True])[email protected]("tool_call", [False, True])+async def test_midstream_read_timeout_is_classified_without_replay(+ *, with_request: bool, async_mode: bool, tool_call: bool+) -> None:+ request = httpx.Request("POST", "https://api.fireworks.ai/inference/v1")+ error = httpx.ReadTimeout(+ "Timeout on reading data from socket",+ request=request if with_request else None,+ )+ delta: dict[str, object] = (+ {+ "tool_calls": [+ {+ "index": 0,+ "id": "call_1",+ "type": "function",+ "function": {"name": "lookup", "arguments": '{"query":'},+ }+ ]+ }+ if tool_call+ else {"content": "Hello"}+ )+ chunk: dict[str, object] = {"choices": [{"delta": delta, "index": 0}]}++ def _stream() -> Iterator[dict[str, object]]:+ yield chunk+ raise error++ async def _astream() -> AsyncIterator[dict[str, object]]:+ yield chunk+ raise error++ model = _make_llm(max_retries=2)+ received: list[AIMessageChunk] = []+ with pytest.raises(httpx.ReadTimeout) as exc_info:+ if async_mode:+ model.async_client = MagicMock(create=AsyncMock(return_value=_astream()))+ async for message in model.astream("Hello"):+ received.append(message) # noqa: PERF401+ else:+ model.client = MagicMock(create=MagicMock(return_value=_stream()))+ received.extend(model.stream("Hello"))++ classified = exc_info.value+ assert isinstance(classified, ModelTimeoutError)+ assert classified.is_retryable+ assert classified.__cause__ is error+ assert str(classified) == str(error)+ if with_request:+ assert classified.request is request+ else:+ with pytest.raises(RuntimeError, match="request"):+ _ = classified.request+ assert len(received) == 1+ if tool_call:+ assert received[0].tool_call_chunks[0]["args"] == '{"query":'+ else:+ assert received[0].content == "Hello"+ client = model.async_client if async_mode else model.client+ client.create.assert_called_once()++[email protected]("async_mode", [False, True])+async def test_midstream_non_timeout_propagates_unchanged(*, async_mode: bool) -> None:+ error = ValueError("invalid stream")++ def _stream() -> Iterator[dict[str, object]]:+ yield {"choices": [{"delta": {"content": "Hello"}}]}+ raise error++ async def _astream() -> AsyncIterator[dict[str, object]]:+ for chunk in _stream():+ yield chunk++ model = _make_llm(max_retries=2)+ with pytest.raises(ValueError) as exc_info:+ if async_mode:+ model.async_client = MagicMock(create=AsyncMock(return_value=_astream()))+ _ = [chunk async for chunk in model.astream("Hello")]+ else:+ model.client = MagicMock(create=MagicMock(return_value=_stream()))+ list(model.stream("Hello"))+ assert exc_info.value is error+ client = model.async_client if async_mode else model.client+ client.create.assert_called_once()++class TestStreamUsage:"""Tests for the `stream_usage` field and `stream_options` plumbing."""