Skip to content

"Vibe-Trading: Your Personal Trading Agent"

fix(providers): interrupt silent provider streams and stop replaying them

Yufeng He35a5385c0232
diff --git a/agent/src/providers/chat.py b/agent/src/providers/chat.py
index 74a9ee744..0c4113184 100644
--- a/agent/src/providers/chat.py
+++ b/agent/src/providers/chat.py
@@ -11,13 +11,13 @@ import inspect
import logging
import os
import re
-import time as _time
from dataclasses import dataclass, field
from typing import Any, Callable, Dict, List, Optional
from src.config.accessor import get_env_config
from src.providers.content_filter import is_content_filter_triggered
from src.providers.llm import build_llm
+from src.providers.stream_guard import StreamInterrupted, supervised_pulls
logger = logging.getLogger(__name__)
@@ -519,17 +519,14 @@ class ChatLLM:
possible_dsml_text = True
cancelled = False
cancelled_usage = None
- last_chunk_ts = _time.monotonic()
- for chunk in llm.stream(messages, config=config, **call_kwargs):
- now = _time.monotonic()
- if idle_timeout_s and now - last_chunk_ts > idle_timeout_s:
- # No delta for too long: the provider is stalled, not
- # thinking. Raising a bare TimeoutError lets the wrapper
- # below convert it into a retryable ProviderStreamError.
- raise TimeoutError(
- f"no stream delta for {idle_timeout_s:.0f}s"
- )
- last_chunk_ts = now
+ interrupted: Optional["StreamInterrupted"] = None
+ stream_chunks = supervised_pulls(
+ llm.stream(messages, config=config, **call_kwargs),
+ idle_timeout_s=idle_timeout_s,
+ total_timeout_s=timeout,
+ should_cancel=should_cancel,
+ )
+ for chunk in stream_chunks:
if should_cancel and should_cancel():
cancelled = True
# This chunk already arrived. Preserve any reported usage
@@ -557,7 +554,10 @@ class ChatLLM:
on_reasoning_chunk(reasoning)
accumulated = chunk if accumulated is None else accumulated + chunk
if accumulated is None:
- if cancelled:
+ if cancelled or interrupted is not None:
+ # Stopping is not a compatibility case: falling back to a
+ # plain invoke here would resend the request the caller
+ # just asked to abandon.
return LLMResponse(
content="", tool_calls=[], finish_reason="stop",
usage_metadata=cancelled_usage,
@@ -583,8 +583,31 @@ class ChatLLM:
if pending_text and not (response.has_tool_calls and response.content == ""):
on_text_chunk(pending_text)
return response
+ except StreamInterrupted as stop:
+ interrupted = stop
+ if stop.reason == "cancelled":
+ # A user stop is not an outage. Hand back whatever arrived
+ # rather than raising, and never fall back to a second request.
+ if cancelled_usage is None and getattr(stop.pending, "usage_metadata", None):
+ cancelled_usage = stop.pending.usage_metadata
+ response = (
+ self._parse_response(accumulated)
+ if accumulated is not None
+ else LLMResponse(content="", tool_calls=[], finish_reason="stop")
+ )
+ if cancelled_usage is not None:
+ response.usage_metadata = cancelled_usage
+ return response
+ _cfg = get_env_config()
+ provider = _cfg.llm.langchain_provider.strip().lower() or "openai"
+ model = self.model_name or _cfg.llm.langchain_model_name.strip() or "(unset)"
+ raise ProviderStreamError(
+ provider=provider, model=model,
+ original=TimeoutError(f"provider stream stopped: {stop.reason}"),
+ usage_metadata=getattr(accumulated, "usage_metadata", None),
+ ) from stop
except Exception as exc:
- if _is_no_stream_chunk_error(exc):
+ if _is_no_stream_chunk_error(exc) and interrupted is None and not cancelled:
# Provider streamed zero chunks (endpoint lacks real SSE); a
# non-streaming invoke still returns a valid response.
logger.warning(
diff --git a/agent/src/providers/stream_guard.py b/agent/src/providers/stream_guard.py
new file mode 100644
index 000000000..ea9462035
--- /dev/null
+++ b/agent/src/providers/stream_guard.py
@@ -0,0 +1,167 @@
+"""Supervise a provider stream whose next chunk may never arrive.
+
+A synchronous provider iterator blocks inside ``next()`` while the wire is
+silent, so a caller that polls a timeout or a cancel flag *after* each chunk
+never gets to poll at all. This races the pull against the idle deadline, the
+total deadline and a cancel predicate, and when that race is lost it closes the
+iterator before reporting the interruption.
+
+Closing is the point. Abandoning the pull and returning would leave the
+transport open and make the caller believe the stream stopped when only its
+consumer did.
+"""
+
+from __future__ import annotations
+
+import queue
+import threading
+import time
+from typing import Any, Callable, Iterator, Optional
+
+#: Sentinel for "the iterator is exhausted".
+_EXHAUSTED = object()
+
+#: How long a close is given to take effect before it is left to the adapter.
+_CLOSE_JOIN_SECONDS = 2.0
+
+
+class StreamInterrupted(Exception):
+ """A stream was stopped before it delivered a usable answer.
+
+ Carries the reason so the caller can tell a user stop from a stall, and
+ whether any partial output was already handed out, because that decides
+ whether a fallback request would replay a turn the user has already seen.
+ """
+
+ def __init__(self, reason: str, *, emitted: bool = False, pending: Any = None) -> None:
+ super().__init__(reason)
+ self.reason = reason
+ self.emitted = emitted
+ #: A chunk the provider had already produced when the stream stopped.
+ #: Its usage was billed, so the caller keeps it even though its text
+ #: and tool calls are dropped.
+ self.pending = pending
+
+
+class _Puller:
+ """Move items off a blocking iterator one at a time onto a queue."""
+
+ def __init__(self, iterator: Iterator[Any], out: "queue.Queue[Any]") -> None:
+ self._iterator = iterator
+ self._out = out
+ self._stop = threading.Event()
+ self._thread = threading.Thread(target=self._run, daemon=True, name="stream-pull")
+ self._thread.start()
+
+ def _run(self) -> None:
+ while not self._stop.is_set():
+ try:
+ item = next(self._iterator)
+ except StopIteration:
+ self._put(_EXHAUSTED)
+ return
+ except BaseException as exc: # noqa: BLE001 - forwarded to the caller
+ self._put(exc)
+ return
+ self._put(item)
+
+ def _put(self, item: Any) -> None:
+ try:
+ self._out.put(item, timeout=1.0)
+ except queue.Full:
+ return
+
+ def stop(self) -> None:
+ self._stop.set()
+
+ def close_iterator(self) -> None:
+ """Close the underlying iterator, bounded so a busy generator cannot hang us."""
+ close = getattr(self._iterator, "close", None)
+ if not callable(close):
+ return
+ # A generator that is mid-execution cannot be closed from here; that is
+ # a limitation of the synchronous iterator protocol, not a decision to
+ # leave the transport open, so it is attempted and bounded either way.
+ done = threading.Event()
+
+ def _attempt() -> None:
+ try:
+ close()
+ except BaseException: # noqa: BLE001 - best effort release
+ pass
+ finally:
+ done.set()
+
+ worker = threading.Thread(target=_attempt, daemon=True, name="stream-close")
+ worker.start()
+ done.wait(_CLOSE_JOIN_SECONDS)
+
+
+def supervised_pulls(
+ iterator: Iterator[Any],
+ *,
+ idle_timeout_s: Optional[float] = None,
+ total_timeout_s: Optional[float] = None,
+ should_cancel: Optional[Callable[[], bool]] = None,
+ poll_interval_s: float = 0.05,
+) -> Iterator[Any]:
+ """Yield items while the stream can still make progress.
+
+ Raises :class:`StreamInterrupted` when the idle deadline passes with no
+ chunk, the total deadline expires, or ``should_cancel`` turns true. The
+ iterator is closed before the exception leaves, and any item that was
+ already queued is discarded rather than delivered late.
+ """
+ if idle_timeout_s is None and total_timeout_s is None and should_cancel is None:
+ yield from iterator
+ return
+
+ out: "queue.Queue[Any]" = queue.Queue(maxsize=1)
+ puller = _Puller(iterator, out)
+ started = time.monotonic()
+ last_chunk = started
+ emitted = False
+
+ def interrupt(reason: str) -> StreamInterrupted:
+ # Take whatever the provider already produced: it was billed, so its
+ # usage belongs on the response even though its text is dropped.
+ try:
+ pending = out.get_nowait()
+ except queue.Empty:
+ pending = None
+ if pending is _EXHAUSTED or isinstance(pending, BaseException):
+ pending = None
+ return StreamInterrupted(reason, emitted=emitted, pending=pending)
+
+ try:
+ while True:
+ now = time.monotonic()
+ remaining: Optional[float] = None
+ if idle_timeout_s is not None:
+ remaining = max(0.0, idle_timeout_s - (now - last_chunk))
+ if total_timeout_s is not None:
+ budget = max(0.0, total_timeout_s - (now - started))
+ remaining = budget if remaining is None else min(remaining, budget)
+ if remaining is not None and remaining <= 0:
+ raise interrupt("idle timeout" if idle_timeout_s is not None else "total timeout")
+ wait = poll_interval_s if remaining is None else min(poll_interval_s, remaining)
+ try:
+ item = out.get(timeout=wait)
+ except queue.Empty:
+ # Nothing arrived in this window, so the only thing that can
+ # still make progress is stopping. A chunk that has already
+ # arrived is always preferred, because it was already billed
+ # and the consumer keeps its usage even while dropping text.
+ if should_cancel is not None and should_cancel():
+ raise interrupt("cancelled")
+ continue
+ if item is _EXHAUSTED:
+ return
+ if isinstance(item, BaseException):
+ raise item
+ last_chunk = time.monotonic()
+ emitted = True
+ yield item
+ finally:
+ puller.stop()
+ puller.close_iterator()
diff --git a/agent/tests/test_chat_llm_streaming.py b/agent/tests/test_chat_llm_streaming.py
index f632bcb45..7d9fa94c6 100644
--- a/agent/tests/test_chat_llm_streaming.py
+++ b/agent/tests/test_chat_llm_streaming.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import os
+import threading
from typing import Any
from unittest.mock import patch
@@ -178,17 +179,22 @@ def test_should_cancel_stops_stream_early() -> None:
_FakeChunk(content="c"),
])
seen: list[str] = []
- calls = {"n": 0}
+ cancelled = threading.Event()
def should_cancel() -> bool:
- # Polled at the top of each chunk: let the first through, cancel after.
- n = calls["n"]
- calls["n"] += 1
- return n >= 1
+ # Let the first chunk through, then stop. Signalled from the delivery
+ # callback rather than counted here: the supervisor polls this while a
+ # stream is silent, so a call-counting predicate would fire before the
+ # first chunk purely from the polling cadence.
+ return cancelled.is_set()
+
+ def on_text(text: str) -> None:
+ seen.append(text)
+ cancelled.set()
response = _client(fake).stream_chat(
[{"role": "user", "content": "hi"}],
- on_text_chunk=seen.append,
+ on_text_chunk=on_text,
should_cancel=should_cancel,
)
diff --git a/agent/tests/test_chat_stream_interruption.py b/agent/tests/test_chat_stream_interruption.py
new file mode 100644
index 000000000..a0b996495
--- /dev/null
+++ b/agent/tests/test_chat_stream_interruption.py
@@ -0,0 +1,184 @@
+"""A silent provider stream must be interruptible and must release its transport.
+
+The old loop checked the idle timeout and the cancel predicate only *after* a
+chunk arrived, so a provider that stopped sending held the caller forever and a
+stop request could not reach it. These tests pin the supervision contract: the
+wait for the next chunk is raced against the deadline and the cancel signal,
+and stopping really closes the stream rather than abandoning a thread.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any
+
+import pytest
+
+from src.providers.chat import ChatLLM
+
+
+class _Chunk:
+ def __init__(self, content: str = "", finish_reason: str = "stop") -> None:
+ self.content = content
+ self.tool_calls: list[dict[str, Any]] = []
+ self.additional_kwargs: dict[str, Any] = {}
+ self.response_metadata = {"finish_reason": finish_reason}
+ self.usage_metadata = None
+
+ def __add__(self, other: "_Chunk") -> "_Chunk":
+ return _Chunk(self.content + other.content, other.response_metadata.get("finish_reason", "stop"))
+
+
+class _SilentStream:
+ """A stream whose first chunk only arrives when the test releases it."""
+
+ def __init__(self, *, chunk: _Chunk | None = None, hold: float = 3.0) -> None:
+ self.chunk = chunk
+ self.hold = hold
+ self.closed = False
+ self.close_count = 0
+ self.released = threading.Event()
+
+ def __iter__(self):
+ return self
+
+ def __next__(self) -> _Chunk:
+ if self.closed:
+ raise StopIteration
+ if not self.released.wait(self.hold):
+ raise StopIteration
+ if self.closed:
+ raise StopIteration
+ if self.chunk is None:
+ raise StopIteration
+ chunk, self.chunk = self.chunk, None
+ return chunk
+
+ def close(self) -> None:
+ self.closed = True
+ self.close_count += 1
+ self.released.set()
+
+
+class _FakeLLM:
+ def __init__(self, stream: _SilentStream) -> None:
+ self.stream_obj = stream
+ self.invoke_called = 0
+
+ def bind_tools(self, tools):
+ return self
+
+ def stream(self, messages, config=None, **kwargs):
+ return self.stream_obj
+
+ def invoke(self, messages, config=None, **kwargs):
+ self.invoke_called += 1
+ return _Chunk("fallback")
+
+
+def _client(stream: _SilentStream) -> ChatLLM:
+ client = ChatLLM.__new__(ChatLLM)
+ client.model_name = "fake"
+ client._llm = _FakeLLM(stream)
+ return client
+
+
+def test_idle_timeout_interrupts_a_silent_stream() -> None:
+ stream = _SilentStream()
+ started = time.monotonic()
+
+ with pytest.raises(Exception):
+ _client(stream).stream_chat(
+ [{"role": "user", "content": "hi"}],
+ idle_timeout_s=0.15,
+ )
+
+ assert time.monotonic() - started < 5
+ assert stream.closed, "the stream must be closed, not merely abandoned"
+
+
+def test_cancel_interrupts_a_silent_stream() -> None:
+ stream = _SilentStream()
+ cancel = threading.Event()
+ threading.Timer(0.15, cancel.set).start()
+ started = time.monotonic()
+
+ response = _client(stream).stream_chat(
+ [{"role": "user", "content": "hi"}],
+ should_cancel=cancel.is_set,
+ )
+
+ assert time.monotonic() - started < 5
+ assert stream.closed
+ assert response.content == ""
+
+
+def test_total_timeout_interrupts_a_silent_stream() -> None:
+ stream = _SilentStream()
+ started = time.monotonic()
+
+ with pytest.raises(Exception):
+ _client(stream).stream_chat(
+ [{"role": "user", "content": "hi"}],
+ timeout=1,
+ )
+
+ assert time.monotonic() - started < 5
+ assert stream.closed
+
+
+def test_no_replay_after_an_interrupted_stream() -> None:
+ """A cancelled or timed-out stream must not fall back and resend the request."""
+ stream = _SilentStream()
+ client = _client(stream)
+ cancel = threading.Event()
+ threading.Timer(0.1, cancel.set).start()
+
+ client.stream_chat([{"role": "user", "content": "hi"}], should_cancel=cancel.is_set)
+
+ assert client._llm.invoke_called == 0
+
+
+def test_no_replay_after_partial_output() -> None:
+ """Once text has been emitted a fallback invoke would replay the turn."""
+ stream = _SilentStream(chunk=_Chunk("partial"))
+ client = _client(stream)
+
+ with pytest.raises(Exception):
+ client.stream_chat(
+ [{"role": "user", "content": "hi"}],
+ on_text_chunk=lambda text: None,
+ idle_timeout_s=0.2,
+ )
+
+ assert client._llm.invoke_called == 0
+
+
+def test_a_late_chunk_after_cancel_never_reaches_the_caller() -> None:
+ stream = _SilentStream()
+ client = _client(stream)
+ cancel = threading.Event()
+ threading.Timer(0.1, cancel.set).start()
+ seen: list[str] = []
+
+ client.stream_chat(
+ [{"role": "user", "content": "hi"}],
+ on_text_chunk=seen.append,
+ should_cancel=cancel.is_set,
+ )
+ stream.released.set()
+ time.sleep(0.2)
+
+ assert seen == []
+
+
+def test_a_delivered_stream_still_reports_its_answer() -> None:
+ stream = _SilentStream(chunk=_Chunk("hello"))
+ stream.released.set()
+ client = _client(stream)
+
+ response = client.stream_chat([{"role": "user", "content": "hi"}], idle_timeout_s=5)
+
+ assert response.content == "hello"
+ assert client._llm.invoke_called == 0