"Vibe-Trading: Your Personal Trading Agent"
Python ai-agentalgorithmic-tradingbacktestingfintechllmmcpmulti-agentpythonquantitative-financetrading
fix(providers): interrupt silent provider streams and stop replaying them
Yufeng He35a5385c0232
diff --git a/agent/src/providers/chat.py b/agent/src/providers/chat.pyindex 74a9ee744..0c4113184 100644--- a/agent/src/providers/chat.py+++ b/agent/src/providers/chat.py@@ -11,13 +11,13 @@ import inspectimport loggingimport osimport re-import time as _timefrom dataclasses import dataclass, fieldfrom typing import Any, Callable, Dict, List, Optionalfrom src.config.accessor import get_env_configfrom src.providers.content_filter import is_content_filter_triggeredfrom src.providers.llm import build_llm+from src.providers.stream_guard import StreamInterrupted, supervised_pullslogger = logging.getLogger(__name__)@@ -519,17 +519,14 @@ class ChatLLM:possible_dsml_text = Truecancelled = Falsecancelled_usage = None- last_chunk_ts = _time.monotonic()- for chunk in llm.stream(messages, config=config, **call_kwargs):- now = _time.monotonic()- if idle_timeout_s and now - last_chunk_ts > idle_timeout_s:- # No delta for too long: the provider is stalled, not- # thinking. Raising a bare TimeoutError lets the wrapper- # below convert it into a retryable ProviderStreamError.- raise TimeoutError(- f"no stream delta for {idle_timeout_s:.0f}s"- )- last_chunk_ts = now+ interrupted: Optional["StreamInterrupted"] = None+ stream_chunks = supervised_pulls(+ llm.stream(messages, config=config, **call_kwargs),+ idle_timeout_s=idle_timeout_s,+ total_timeout_s=timeout,+ should_cancel=should_cancel,+ )+ for chunk in stream_chunks:if should_cancel and should_cancel():cancelled = True# This chunk already arrived. Preserve any reported usage@@ -557,7 +554,10 @@ class ChatLLM:on_reasoning_chunk(reasoning)accumulated = chunk if accumulated is None else accumulated + chunkif accumulated is None:- if cancelled:+ if cancelled or interrupted is not None:+ # Stopping is not a compatibility case: falling back to a+ # plain invoke here would resend the request the caller+ # just asked to abandon.return LLMResponse(content="", tool_calls=[], finish_reason="stop",usage_metadata=cancelled_usage,@@ -583,8 +583,31 @@ class ChatLLM:if pending_text and not (response.has_tool_calls and response.content == ""):on_text_chunk(pending_text)return response+ except StreamInterrupted as stop:+ interrupted = stop+ if stop.reason == "cancelled":+ # A user stop is not an outage. Hand back whatever arrived+ # rather than raising, and never fall back to a second request.+ if cancelled_usage is None and getattr(stop.pending, "usage_metadata", None):+ cancelled_usage = stop.pending.usage_metadata+ response = (+ self._parse_response(accumulated)+ if accumulated is not None+ else LLMResponse(content="", tool_calls=[], finish_reason="stop")+ )+ if cancelled_usage is not None:+ response.usage_metadata = cancelled_usage+ return response+ _cfg = get_env_config()+ provider = _cfg.llm.langchain_provider.strip().lower() or "openai"+ model = self.model_name or _cfg.llm.langchain_model_name.strip() or "(unset)"+ raise ProviderStreamError(+ provider=provider, model=model,+ original=TimeoutError(f"provider stream stopped: {stop.reason}"),+ usage_metadata=getattr(accumulated, "usage_metadata", None),+ ) from stopexcept Exception as exc:- if _is_no_stream_chunk_error(exc):+ if _is_no_stream_chunk_error(exc) and interrupted is None and not cancelled:# Provider streamed zero chunks (endpoint lacks real SSE); a# non-streaming invoke still returns a valid response.logger.warning(diff --git a/agent/src/providers/stream_guard.py b/agent/src/providers/stream_guard.pynew file mode 100644index 000000000..ea9462035--- /dev/null+++ b/agent/src/providers/stream_guard.py@@ -0,0 +1,167 @@+"""Supervise a provider stream whose next chunk may never arrive.++A synchronous provider iterator blocks inside ``next()`` while the wire is+silent, so a caller that polls a timeout or a cancel flag *after* each chunk+never gets to poll at all. This races the pull against the idle deadline, the+total deadline and a cancel predicate, and when that race is lost it closes the+iterator before reporting the interruption.++Closing is the point. Abandoning the pull and returning would leave the+transport open and make the caller believe the stream stopped when only its+consumer did.+"""++from __future__ import annotations++import queue+import threading+import time+from typing import Any, Callable, Iterator, Optional++#: Sentinel for "the iterator is exhausted".+_EXHAUSTED = object()++#: How long a close is given to take effect before it is left to the adapter.+_CLOSE_JOIN_SECONDS = 2.0+++class StreamInterrupted(Exception):+ """A stream was stopped before it delivered a usable answer.++ Carries the reason so the caller can tell a user stop from a stall, and+ whether any partial output was already handed out, because that decides+ whether a fallback request would replay a turn the user has already seen.+ """++ def __init__(self, reason: str, *, emitted: bool = False, pending: Any = None) -> None:+ super().__init__(reason)+ self.reason = reason+ self.emitted = emitted+ #: A chunk the provider had already produced when the stream stopped.+ #: Its usage was billed, so the caller keeps it even though its text+ #: and tool calls are dropped.+ self.pending = pending+++class _Puller:+ """Move items off a blocking iterator one at a time onto a queue."""++ def __init__(self, iterator: Iterator[Any], out: "queue.Queue[Any]") -> None:+ self._iterator = iterator+ self._out = out+ self._stop = threading.Event()+ self._thread = threading.Thread(target=self._run, daemon=True, name="stream-pull")+ self._thread.start()++ def _run(self) -> None:+ while not self._stop.is_set():+ try:+ item = next(self._iterator)+ except StopIteration:+ self._put(_EXHAUSTED)+ return+ except BaseException as exc: # noqa: BLE001 - forwarded to the caller+ self._put(exc)+ return+ self._put(item)++ def _put(self, item: Any) -> None:+ try:+ self._out.put(item, timeout=1.0)+ except queue.Full:+ return++ def stop(self) -> None:+ self._stop.set()++ def close_iterator(self) -> None:+ """Close the underlying iterator, bounded so a busy generator cannot hang us."""+ close = getattr(self._iterator, "close", None)+ if not callable(close):+ return+ # A generator that is mid-execution cannot be closed from here; that is+ # a limitation of the synchronous iterator protocol, not a decision to+ # leave the transport open, so it is attempted and bounded either way.+ done = threading.Event()++ def _attempt() -> None:+ try:+ close()+ except BaseException: # noqa: BLE001 - best effort release+ pass+ finally:+ done.set()++ worker = threading.Thread(target=_attempt, daemon=True, name="stream-close")+ worker.start()+ done.wait(_CLOSE_JOIN_SECONDS)+++def supervised_pulls(+ iterator: Iterator[Any],+ *,+ idle_timeout_s: Optional[float] = None,+ total_timeout_s: Optional[float] = None,+ should_cancel: Optional[Callable[[], bool]] = None,+ poll_interval_s: float = 0.05,+) -> Iterator[Any]:+ """Yield items while the stream can still make progress.++ Raises :class:`StreamInterrupted` when the idle deadline passes with no+ chunk, the total deadline expires, or ``should_cancel`` turns true. The+ iterator is closed before the exception leaves, and any item that was+ already queued is discarded rather than delivered late.+ """+ if idle_timeout_s is None and total_timeout_s is None and should_cancel is None:+ yield from iterator+ return++ out: "queue.Queue[Any]" = queue.Queue(maxsize=1)+ puller = _Puller(iterator, out)+ started = time.monotonic()+ last_chunk = started+ emitted = False++ def interrupt(reason: str) -> StreamInterrupted:+ # Take whatever the provider already produced: it was billed, so its+ # usage belongs on the response even though its text is dropped.+ try:+ pending = out.get_nowait()+ except queue.Empty:+ pending = None+ if pending is _EXHAUSTED or isinstance(pending, BaseException):+ pending = None+ return StreamInterrupted(reason, emitted=emitted, pending=pending)++ try:+ while True:+ now = time.monotonic()+ remaining: Optional[float] = None+ if idle_timeout_s is not None:+ remaining = max(0.0, idle_timeout_s - (now - last_chunk))+ if total_timeout_s is not None:+ budget = max(0.0, total_timeout_s - (now - started))+ remaining = budget if remaining is None else min(remaining, budget)+ if remaining is not None and remaining <= 0:+ raise interrupt("idle timeout" if idle_timeout_s is not None else "total timeout")+ wait = poll_interval_s if remaining is None else min(poll_interval_s, remaining)+ try:+ item = out.get(timeout=wait)+ except queue.Empty:+ # Nothing arrived in this window, so the only thing that can+ # still make progress is stopping. A chunk that has already+ # arrived is always preferred, because it was already billed+ # and the consumer keeps its usage even while dropping text.+ if should_cancel is not None and should_cancel():+ raise interrupt("cancelled")+ continue+ if item is _EXHAUSTED:+ return+ if isinstance(item, BaseException):+ raise item+ last_chunk = time.monotonic()+ emitted = True+ yield item+ finally:+ puller.stop()+ puller.close_iterator()diff --git a/agent/tests/test_chat_llm_streaming.py b/agent/tests/test_chat_llm_streaming.pyindex f632bcb45..7d9fa94c6 100644--- a/agent/tests/test_chat_llm_streaming.py+++ b/agent/tests/test_chat_llm_streaming.py@@ -3,6 +3,7 @@from __future__ import annotationsimport os+import threadingfrom typing import Anyfrom unittest.mock import patch@@ -178,17 +179,22 @@ def test_should_cancel_stops_stream_early() -> None:_FakeChunk(content="c"),])seen: list[str] = []- calls = {"n": 0}+ cancelled = threading.Event()def should_cancel() -> bool:- # Polled at the top of each chunk: let the first through, cancel after.- n = calls["n"]- calls["n"] += 1- return n >= 1+ # Let the first chunk through, then stop. Signalled from the delivery+ # callback rather than counted here: the supervisor polls this while a+ # stream is silent, so a call-counting predicate would fire before the+ # first chunk purely from the polling cadence.+ return cancelled.is_set()++ def on_text(text: str) -> None:+ seen.append(text)+ cancelled.set()response = _client(fake).stream_chat([{"role": "user", "content": "hi"}],- on_text_chunk=seen.append,+ on_text_chunk=on_text,should_cancel=should_cancel,)diff --git a/agent/tests/test_chat_stream_interruption.py b/agent/tests/test_chat_stream_interruption.pynew file mode 100644index 000000000..a0b996495--- /dev/null+++ b/agent/tests/test_chat_stream_interruption.py@@ -0,0 +1,184 @@+"""A silent provider stream must be interruptible and must release its transport.++The old loop checked the idle timeout and the cancel predicate only *after* a+chunk arrived, so a provider that stopped sending held the caller forever and a+stop request could not reach it. These tests pin the supervision contract: the+wait for the next chunk is raced against the deadline and the cancel signal,+and stopping really closes the stream rather than abandoning a thread.+"""++from __future__ import annotations++import threading+import time+from typing import Any++import pytest++from src.providers.chat import ChatLLM+++class _Chunk:+ def __init__(self, content: str = "", finish_reason: str = "stop") -> None:+ self.content = content+ self.tool_calls: list[dict[str, Any]] = []+ self.additional_kwargs: dict[str, Any] = {}+ self.response_metadata = {"finish_reason": finish_reason}+ self.usage_metadata = None++ def __add__(self, other: "_Chunk") -> "_Chunk":+ return _Chunk(self.content + other.content, other.response_metadata.get("finish_reason", "stop"))+++class _SilentStream:+ """A stream whose first chunk only arrives when the test releases it."""++ def __init__(self, *, chunk: _Chunk | None = None, hold: float = 3.0) -> None:+ self.chunk = chunk+ self.hold = hold+ self.closed = False+ self.close_count = 0+ self.released = threading.Event()++ def __iter__(self):+ return self++ def __next__(self) -> _Chunk:+ if self.closed:+ raise StopIteration+ if not self.released.wait(self.hold):+ raise StopIteration+ if self.closed:+ raise StopIteration+ if self.chunk is None:+ raise StopIteration+ chunk, self.chunk = self.chunk, None+ return chunk++ def close(self) -> None:+ self.closed = True+ self.close_count += 1+ self.released.set()+++class _FakeLLM:+ def __init__(self, stream: _SilentStream) -> None:+ self.stream_obj = stream+ self.invoke_called = 0++ def bind_tools(self, tools):+ return self++ def stream(self, messages, config=None, **kwargs):+ return self.stream_obj++ def invoke(self, messages, config=None, **kwargs):+ self.invoke_called += 1+ return _Chunk("fallback")+++def _client(stream: _SilentStream) -> ChatLLM:+ client = ChatLLM.__new__(ChatLLM)+ client.model_name = "fake"+ client._llm = _FakeLLM(stream)+ return client+++def test_idle_timeout_interrupts_a_silent_stream() -> None:+ stream = _SilentStream()+ started = time.monotonic()++ with pytest.raises(Exception):+ _client(stream).stream_chat(+ [{"role": "user", "content": "hi"}],+ idle_timeout_s=0.15,+ )++ assert time.monotonic() - started < 5+ assert stream.closed, "the stream must be closed, not merely abandoned"+++def test_cancel_interrupts_a_silent_stream() -> None:+ stream = _SilentStream()+ cancel = threading.Event()+ threading.Timer(0.15, cancel.set).start()+ started = time.monotonic()++ response = _client(stream).stream_chat(+ [{"role": "user", "content": "hi"}],+ should_cancel=cancel.is_set,+ )++ assert time.monotonic() - started < 5+ assert stream.closed+ assert response.content == ""+++def test_total_timeout_interrupts_a_silent_stream() -> None:+ stream = _SilentStream()+ started = time.monotonic()++ with pytest.raises(Exception):+ _client(stream).stream_chat(+ [{"role": "user", "content": "hi"}],+ timeout=1,+ )++ assert time.monotonic() - started < 5+ assert stream.closed+++def test_no_replay_after_an_interrupted_stream() -> None:+ """A cancelled or timed-out stream must not fall back and resend the request."""+ stream = _SilentStream()+ client = _client(stream)+ cancel = threading.Event()+ threading.Timer(0.1, cancel.set).start()++ client.stream_chat([{"role": "user", "content": "hi"}], should_cancel=cancel.is_set)++ assert client._llm.invoke_called == 0+++def test_no_replay_after_partial_output() -> None:+ """Once text has been emitted a fallback invoke would replay the turn."""+ stream = _SilentStream(chunk=_Chunk("partial"))+ client = _client(stream)++ with pytest.raises(Exception):+ client.stream_chat(+ [{"role": "user", "content": "hi"}],+ on_text_chunk=lambda text: None,+ idle_timeout_s=0.2,+ )++ assert client._llm.invoke_called == 0+++def test_a_late_chunk_after_cancel_never_reaches_the_caller() -> None:+ stream = _SilentStream()+ client = _client(stream)+ cancel = threading.Event()+ threading.Timer(0.1, cancel.set).start()+ seen: list[str] = []++ client.stream_chat(+ [{"role": "user", "content": "hi"}],+ on_text_chunk=seen.append,+ should_cancel=cancel.is_set,+ )+ stream.released.set()+ time.sleep(0.2)++ assert seen == []+++def test_a_delivered_stream_still_reports_its_answer() -> None:+ stream = _SilentStream(chunk=_Chunk("hello"))+ stream.released.set()+ client = _client(stream)++ response = client.stream_chat([{"role": "user", "content": "hi"}], idle_timeout_s=5)++ assert response.content == "hello"+ assert client._llm.invoke_called == 0