Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend/copilot): close empty-completion gaps + add silence heartbeat (#13056)

Ubbedce0cd9a4f23
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service.py b/autogpt_platform/backend/backend/copilot/baseline/service.py
index 5328ef7fab..6ebcce28ff 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service.py
@@ -148,6 +148,35 @@ _BUDGET_EXHAUSTED_FALLBACK_TEXT = (
"Send a follow-up message to continue from here."
)
+# Fallback surfaced when the model finished naturally (under the budget) but
+# left the terminal round with zero visible text — the baseline equivalent
+# of the SDK's ``StreamError(code="empty_completion")``. Without this the FE
+# silently shows nothing after a long thinking-only turn on baseline-routed
+# sessions (e.g. some Kimi K2.6 cohorts).
+_NATURAL_FINISH_EMPTY_FALLBACK_TEXT = (
+ "The model returned no response for this turn. "
+ "Please try again or rephrase your request."
+)
+
+
+def _silent_finish_fallback_events(
+ notice: str | None,
+) -> tuple[list[StreamBaseResponse], str]:
+ """Wrap a fallback notice (or ``None``) into the matching wire events.
+
+ Shared by the budget-exhausted and natural-finish-empty branches so the
+ event sequencing stays identical — only the message text differs.
+ """
+ if notice is None:
+ return [], ""
+ block_id = str(uuid.uuid4())
+ events: list[StreamBaseResponse] = [
+ StreamTextStart(id=block_id),
+ StreamTextDelta(id=block_id, delta=notice),
+ StreamTextEnd(id=block_id),
+ ]
+ return events, notice
+
def _budget_exhausted_notice_text(terminal_round_text: str) -> str | None:
"""Return the fallback notice when a budget-exhausted turn produced no
@@ -161,6 +190,15 @@ def _budget_exhausted_notice_text(terminal_round_text: str) -> str | None:
return _BUDGET_EXHAUSTED_FALLBACK_TEXT
+def _natural_finish_empty_notice_text(terminal_round_text: str) -> str | None:
+ """Return the fallback notice when a naturally-finishing turn produced
+ no visible text, or ``None`` when the model already summarised itself.
+ """
+ if terminal_round_text.strip():
+ return None
+ return _NATURAL_FINISH_EMPTY_FALLBACK_TEXT
+
+
def _build_budget_exhausted_fallback_events(
terminal_round_text: str,
) -> tuple[list[StreamBaseResponse], str]:
@@ -171,16 +209,23 @@ def _build_budget_exhausted_fallback_events(
no fallback is needed. Split out of the async generator so it's unit-
testable without the surrounding streaming machinery.
"""
- notice = _budget_exhausted_notice_text(terminal_round_text)
- if notice is None:
- return [], ""
- block_id = str(uuid.uuid4())
- events: list[StreamBaseResponse] = [
- StreamTextStart(id=block_id),
- StreamTextDelta(id=block_id, delta=notice),
- StreamTextEnd(id=block_id),
- ]
- return events, notice
+ return _silent_finish_fallback_events(
+ _budget_exhausted_notice_text(terminal_round_text)
+ )
+
+
+def _build_natural_finish_empty_fallback_events(
+ terminal_round_text: str,
+) -> tuple[list[StreamBaseResponse], str]:
+ """Build the fallback stream events surfaced when a naturally-finishing
+ turn left the terminal round with no visible text.
+
+ Returns ``(events, text_to_append)``. Empty list + empty string when
+ no fallback is needed.
+ """
+ return _silent_finish_fallback_events(
+ _natural_finish_empty_notice_text(terminal_round_text)
+ )
# Max seconds to wait for transcript upload in the finally block before
@@ -2044,6 +2089,21 @@ async def stream_chat_completion_baseline(
for evt in fallback_events:
yield evt
state.assistant_text += fallback_text
+ elif loop_result and loop_result.finished_naturally:
+ # mirrors SDK's empty_completion surface — prevents blank bubble on thinking-only turns
+ terminal_round_text = state.assistant_text[text_len_before_final_round[0] :]
+ fallback_events, fallback_text = (
+ _build_natural_finish_empty_fallback_events(terminal_round_text)
+ )
+ if fallback_events:
+ logger.warning(
+ "[Baseline] Natural finish with empty terminal round (%d "
+ "iterations); surfacing fallback notice",
+ loop_result.iterations,
+ )
+ for evt in fallback_events:
+ yield evt
+ state.assistant_text += fallback_text
except Exception as e:
_stream_error = True
error_msg = str(e) or type(e).__name__
diff --git a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
index f415247f85..80a64fc77b 100644
--- a/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
+++ b/autogpt_platform/backend/backend/copilot/baseline/service_unit_test.py
@@ -11,18 +11,21 @@ from openai.types.chat import ChatCompletionToolParam
from backend.copilot.baseline.service import (
_BUDGET_EXHAUSTED_FALLBACK_TEXT,
+ _NATURAL_FINISH_EMPTY_FALLBACK_TEXT,
_baseline_conversation_updater,
_baseline_llm_caller,
_BaselineStreamState,
_budget_exhausted_notice_text,
_build_budget_exhausted_fallback_events,
_build_cached_system_message,
+ _build_natural_finish_empty_fallback_events,
_compress_session_messages,
_fresh_anthropic_caching_headers,
_fresh_ephemeral_cache_control,
_is_anthropic_model,
_mark_system_message_with_cache_control,
_mark_tools_with_cache_control,
+ _natural_finish_empty_notice_text,
_supports_prompt_cache_markers,
)
from backend.copilot.model import ChatMessage
@@ -2335,6 +2338,54 @@ class TestBuildBudgetExhaustedFallbackEvents:
assert events_a[0].id != events_b[0].id
+class TestNaturalFinishEmptyNoticeText:
+ """Fallback decision when the model finished naturally (under budget)
+ but the terminal round produced no visible text — the baseline
+ equivalent of the SDK's ``empty_completion`` StreamError. Without
+ this fallback the FE shows nothing after a long thinking-only turn
+ on baseline-routed sessions (e.g. some Kimi K2.6 cohorts)."""
+
+ def test_empty_text_returns_fallback(self):
+ assert (
+ _natural_finish_empty_notice_text("") == _NATURAL_FINISH_EMPTY_FALLBACK_TEXT
+ )
+
+ def test_whitespace_only_returns_fallback(self):
+ assert (
+ _natural_finish_empty_notice_text(" \n\t ")
+ == _NATURAL_FINISH_EMPTY_FALLBACK_TEXT
+ )
+
+ def test_non_empty_text_returns_none(self):
+ assert _natural_finish_empty_notice_text("Here is the answer.") is None
+
+ def test_fallback_text_is_user_facing(self):
+ """Guard against shipping an empty / internal string."""
+ assert _NATURAL_FINISH_EMPTY_FALLBACK_TEXT.strip()
+ # Distinct from the budget message — they describe different causes
+ # so the user gets accurate signal about what happened.
+ assert _NATURAL_FINISH_EMPTY_FALLBACK_TEXT != _BUDGET_EXHAUSTED_FALLBACK_TEXT
+
+
+class TestBuildNaturalFinishEmptyFallbackEvents:
+ def test_empty_terminal_text_emits_three_events(self):
+ events, to_append = _build_natural_finish_empty_fallback_events("")
+ assert to_append == _NATURAL_FINISH_EMPTY_FALLBACK_TEXT
+ assert len(events) == 3
+ assert isinstance(events[0], StreamTextStart)
+ assert isinstance(events[1], StreamTextDelta)
+ assert isinstance(events[2], StreamTextEnd)
+ assert events[0].id == events[1].id == events[2].id
+ assert events[1].delta == _NATURAL_FINISH_EMPTY_FALLBACK_TEXT
+
+ def test_non_empty_terminal_text_returns_empty(self):
+ events, to_append = _build_natural_finish_empty_fallback_events(
+ "Here is the answer."
+ )
+ assert events == []
+ assert to_append == ""
+
+
class TestStreamOptionsGating:
"""stream_options must be present for OR and absent for direct Anthropic."""
diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.py
index 61e0daa7b6..fc126847df 100644
--- a/autogpt_platform/backend/backend/copilot/executor/processor.py
+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py
@@ -18,6 +18,7 @@ from backend.copilot.config import ChatConfig, CopilotMode
from backend.copilot.response_model import StreamError
from backend.copilot.sdk import service as sdk_service
from backend.copilot.sdk.dummy import stream_chat_completion_dummy
+from backend.copilot.stream_heartbeat import wrap_stream_with_heartbeat
from backend.executor.cluster_lock import ClusterLock
from backend.util.decorator import error_logged
from backend.util.feature_flag import Flag, is_feature_enabled
@@ -528,10 +529,15 @@ class CoPilotProcessor:
permissions=entry.permissions,
request_arrival_at=entry.request_arrival_at,
)
+ # Surround the driver stream with a silence watchdog so the
+ # FE shows escalating ``StreamStatus`` messages during long
+ # silent gaps (deep thinking, slow tool execution, inter-tool
+ # gaps) instead of looping the generic "Thinking…" phrases.
+ heartbeat_stream = wrap_stream_with_heartbeat(raw_stream)
published_stream = stream_registry.stream_and_publish(
session_id=entry.session_id,
turn_id=entry.turn_id,
- stream=raw_stream,
+ stream=heartbeat_stream,
)
# Explicit aclose() on early exit: ``async for … break`` does
# not close the generator, so GeneratorExit would never reach
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
index 74dd7d121a..4280b1d89f 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter.py
@@ -639,7 +639,7 @@ class SDKResponseAdapter:
"""True when the turn ended with no user-visible content the wire
can render.
- Two failure modes:
+ Three failure modes:
* **Ghost-finished success** (SECRT-2252): a ``subtype="success"``
ResultMessage whose ``result`` is empty, ``output_tokens == 0``,
@@ -661,6 +661,15 @@ class SDKResponseAdapter:
(with non-tool content) triggered an earlier orphan flush — so by
the time the ResultMessage lands, ``current_tool_calls`` looks
fully resolved even though the actual tool_results were synthetic.
+ * **Thinking-only success with no tools**: model emitted only
+ ``ThinkingBlock`` content and ended the turn with
+ ``subtype="success"`` and no ``TextBlock`` / ``ToolUseBlock``.
+ ``_is_empty_completion`` excludes this (reasoning started), and
+ the post-tool-result re-prompt at line 470 requires
+ ``_any_tool_results_seen``. Without surfacing it the FE renders
+ an inline reasoning panel that's visually identical to a
+ still-streaming turn (Toran's prod symptom in the
+ "While testing the new payment, billing" thread, 2026-05-06).
"""
# Specific error subtypes (budget exhaustion, max turns) carry their
# own user-facing reason — don't shadow them with the generic empty-
@@ -682,6 +691,14 @@ class SDKResponseAdapter:
)
if had_orphan_tool_use and self._last_assistant_had_empty_content:
return True
+ if (
+ msg.subtype == "success"
+ and self.has_started_reasoning
+ and not self.has_started_text
+ and not self._any_tool_results_seen
+ and not self.current_tool_calls
+ ):
+ return True
return False
def _is_empty_completion(self, msg: ResultMessage) -> bool:
diff --git a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
index 35d2b37625..a1e8a07b07 100644
--- a/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
+++ b/autogpt_platform/backend/backend/copilot/sdk/response_adapter_test.py
@@ -832,6 +832,73 @@ def test_result_success_with_text_emits_finish_not_error():
assert "StreamError" not in types
+def test_result_success_thinking_only_no_tools_emits_empty_completion():
+ """Thinking-only success with zero tools: model emitted ThinkingBlock(s)
+ and nothing else, ResultMessage subtype="success", no tool_use / tool_result
+ seen. ``_is_empty_completion`` excludes turns where reasoning started, and
+ the post-tool-result re-prompt guard requires ``_any_tool_results_seen``.
+ Without surfacing this, the FE renders an inline expanded reasoning panel
+ indistinguishable from a still-streaming turn (Toran's prod symptom in the
+ "While testing the new payment, billing" thread)."""
+ adapter = _adapter()
+ adapter.convert_message(SystemMessage(subtype="init", data={}))
+ adapter.convert_message(
+ AssistantMessage(
+ content=[
+ ThinkingBlock(thinking="reasoning about the answer", signature="sig")
+ ],
+ model="test",
+ )
+ )
+ msg = ResultMessage(
+ subtype="success",
+ duration_ms=100,
+ duration_api_ms=50,
+ is_error=False,
+ num_turns=1,
+ session_id="s1",
+ result="",
+ usage={"output_tokens": 50},
+ )
+ results = adapter.convert_message(msg)
+ types = [type(r).__name__ for r in results]
+ assert "StreamError" in types
+ assert "StreamFinish" in types
+ assert types.index("StreamError") < types.index("StreamFinish")
+ err = next(r for r in results if isinstance(r, StreamError))
+ assert err.code == "empty_completion"
+
+
+def test_result_success_thinking_only_with_text_after_no_error():
+ """Regression guard: when text follows the ThinkingBlock, the turn is
+ legitimately complete and must NOT trip the new thinking-only guard."""
+ adapter = _adapter()
+ adapter.convert_message(SystemMessage(subtype="init", data={}))
+ adapter.convert_message(
+ AssistantMessage(
+ content=[
+ ThinkingBlock(thinking="figuring out", signature="sig"),
+ TextBlock(text="here is the answer"),
+ ],
+ model="test",
+ )
+ )
+ msg = ResultMessage(
+ subtype="success",
+ duration_ms=100,
+ duration_api_ms=50,
+ is_error=False,
+ num_turns=1,
+ session_id="s1",
+ result="here is the answer",
+ usage={"output_tokens": 50},
+ )
+ results = adapter.convert_message(msg)
+ types = [type(r).__name__ for r in results]
+ assert "StreamFinish" in types
+ assert "StreamError" not in types
+
+
def test_result_success_with_nonzero_output_tokens_not_empty():
"""If ``output_tokens > 0`` but ``result`` is empty, don't classify as
empty — fall through to the existing success path. No prior
diff --git a/autogpt_platform/backend/backend/copilot/stream_heartbeat.py b/autogpt_platform/backend/backend/copilot/stream_heartbeat.py
new file mode 100644
index 0000000000..e4e88e8b78
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/stream_heartbeat.py
@@ -0,0 +1,223 @@
+"""Silence watchdog — keeps the FE informed during long copilot turns.
+
+A copilot turn can stay silent for many seconds at a time:
+
+* deep ``ThinkingBlock`` reasoning before any text/tool chunk lands,
+* slow tool execution (e.g. ``browser_act`` navigation, video gen),
+* inter-tool gaps where the model is computing the next call.
+
+The FE's ``ThinkingIndicator`` cycles generic phrases and surfaces elapsed
+time after 20s, but a backend-emitted ``StreamStatus`` overrides the
+phrase rotation with a more specific message. Existing emissions only
+fire at boundaries ("Contacting the model…", "Analyzing result…"); this
+watchdog fills the silent gaps in between with escalating reassurance.
+
+Usage:
+
+ async def emit(status):
+ await out_queue.put(status)
+
+ async with SilenceWatchdog(emit_status=emit) as watchdog:
+ async for event in driver_stream():
+ if not isinstance(event, StreamStatus):
+ watchdog.bump()
+ else:
+ watchdog.note_status_emitted()
+ yield event
+
+Each non-status yield resets the silence timer; the watchdog only fires
+its escalation messages when the gap between bumps grows past a
+threshold. ``note_status_emitted()`` lets driver-emitted status messages
+suppress the watchdog for a short window so the two don't talk over each
+other.
+"""
+
+import asyncio
+import contextlib
+import logging
+import time
+from collections.abc import AsyncGenerator, Awaitable, Callable
+from typing import Any
+
+from backend.copilot.response_model import StreamBaseResponse, StreamStatus
+
+logger = logging.getLogger(__name__)
+
+
+# Default escalation schedule: (silent_seconds, status_message).
+# Thresholds are measured from the last ``bump()``. Each tier fires
+# once per silent gap; ``bump()`` re-arms the schedule.
+DEFAULT_SCHEDULE: list[tuple[float, str]] = [
+ (10.0, "Working on it…"),
+ (30.0, "Still working — complex requests can take a moment…"),
+ (60.0, "This is taking longer than usual…"),
+]
+
+
+# When the driver itself emits a ``StreamStatus`` (e.g. "Optimizing
+# conversation context…"), the watchdog stays quiet for this many
+# seconds so it doesn't immediately overwrite that message.
+DEFAULT_SUPPRESSION_WINDOW_S = 5.0
+
+
+# How often the internal loop wakes to check thresholds. 1s is fine
+# for human-perceivable cadence and keeps wakeups cheap.
+DEFAULT_TICK_S = 1.0
+
+
+class SilenceWatchdog:
+ """Async context manager that emits ``StreamStatus`` during silence.
+
+ The watchdog is fully passive when there's regular activity — it only
+ fires when ``bump()`` hasn't been called within a threshold.
+ """
+
+ def __init__(
+ self,
+ emit_status: Callable[[StreamStatus], Awaitable[None]],
+ *,
+ schedule: list[tuple[float, str]] | None = None,
+ suppression_window_s: float = DEFAULT_SUPPRESSION_WINDOW_S,
+ tick_s: float = DEFAULT_TICK_S,
+ clock: Callable[[], float] = time.monotonic,
+ ) -> None:
+ self._emit_status = emit_status
+ self._schedule = sorted(schedule if schedule is not None else DEFAULT_SCHEDULE)
+ self._suppression_window_s = suppression_window_s
+ self._tick_s = tick_s
+ self._clock = clock
+
+ now = self._clock()
+ self._last_event_at = now
+ self._last_status_emitted_at: float = float("-inf")
+ # Fired thresholds since the last ``bump()``. Uses index into the
+ # sorted schedule so we can quickly check whether a tier already
+ # fired during the current silent gap.
+ self._fired_tiers: set[int] = set()
+ self._task: asyncio.Task[None] | None = None
+
+ def bump(self) -> None:
+ """Mark a non-status event as just emitted.
+
+ Resets the silence timer and re-arms the escalation schedule.
+ """
+ self._last_event_at = self._clock()
+ self._fired_tiers.clear()
+
+ def note_status_emitted(self) -> None:
+ """Mark that a non-watchdog ``StreamStatus`` was just emitted.
+
+ Activates the suppression window so the watchdog doesn't fire
+ immediately on top of the driver's own status message.
+ """
+ self._last_status_emitted_at = self._clock()
+
+ async def __aenter__(self) -> "SilenceWatchdog":
+ self._task = asyncio.create_task(self._loop(), name="silence-watchdog")
+ return self
+
+ async def __aexit__(self, *_exc: object) -> None:
+ if self._task is None:
+ return
+ self._task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await self._task
+ self._task = None
+
+ async def _loop(self) -> None:
+ """Tick loop. Fires due tiers; sleeps for ``tick_s`` between
+ checks so tests can use sub-second thresholds without flakiness.
+ """
+ try:
+ while True:
+ await asyncio.sleep(self._tick_s)
+ now = self._clock()
+ # Suppression: if a non-watchdog status was emitted recently,
+ # don't fire on top of it.
+ if now - self._last_status_emitted_at < self._suppression_window_s:
+ continue
+ silent_for = now - self._last_event_at
+ for tier_index, (threshold, message) in enumerate(self._schedule):
+ if tier_index in self._fired_tiers:
+ continue
+ if silent_for < threshold:
+ break # schedule is sorted — later tiers are still further out
+ await self._safe_emit(message)
+ self._fired_tiers.add(tier_index)
+ self._last_status_emitted_at = now
+ break # one emission per tick; suppression window paces the next tier
+ except asyncio.CancelledError:
+ raise
+ except Exception: # noqa: BLE001 — defensive: keep loop alive on bugs in emit
+ logger.exception("[SilenceWatchdog] unexpected error in tick loop")
+
+ async def _safe_emit(self, message: str) -> None:
+ """Emit a status, swallowing exceptions so a transient SSE
+ failure doesn't kill the watchdog (the next tier still has a
+ chance to fire if the channel recovers).
+ """
+ try:
+ await self._emit_status(StreamStatus(message=message))
+ except Exception: # noqa: BLE001
+ logger.warning(
+ "[SilenceWatchdog] emit_status raised; continuing", exc_info=True
+ )
+
+
+async def wrap_stream_with_heartbeat(
+ stream: AsyncGenerator[StreamBaseResponse, None],
+ *,
+ schedule: list[tuple[float, str]] | None = None,
+ suppression_window_s: float = DEFAULT_SUPPRESSION_WINDOW_S,
+ tick_s: float = DEFAULT_TICK_S,
+) -> AsyncGenerator[StreamBaseResponse, None]:
+ """Wrap a stream of ``StreamBaseResponse`` events with a silence
+ watchdog so periodic ``StreamStatus`` events fire during long gaps.
+
+ Driver-emitted ``StreamStatus`` events arm the suppression window so
+ the watchdog doesn't talk over them. All other driver events bump the
+ silence timer. Watchdog emissions are interleaved into the output
+ stream via an internal queue so consumers see a single merged stream.
+ """
+ out_queue: asyncio.Queue[tuple[str, Any]] = asyncio.Queue()
+ sentinel = object()
+
+ async def _emit_from_watchdog(status: StreamStatus) -> None:
+ await out_queue.put(("watchdog", status))
+
+ async def _drain_driver() -> None:
+ try:
+ async for event in stream:
+ await out_queue.put(("driver", event))
+ finally:
+ # Forward close to the underlying stream so its own cleanup
+ # (e.g. SDK stream lock release) runs deterministically — the
+ # ``async for`` above does not call ``aclose()`` on cancel /
+ # GeneratorExit, leaving the lock held until GC. Mirrors the
+ # explicit aclose pattern in ``executor/processor.py``.
+ with contextlib.suppress(Exception):
+ await stream.aclose()
+ await out_queue.put(("driver", sentinel))
+
+ drain_task = asyncio.create_task(_drain_driver(), name="stream-heartbeat-drain")
+ try:
+ async with SilenceWatchdog(
+ emit_status=_emit_from_watchdog,
+ schedule=schedule,
+ suppression_window_s=suppression_window_s,
+ tick_s=tick_s,
+ ) as watchdog:
+ while True:
+ source, event = await out_queue.get()
+ if event is sentinel:
+ break
+ if source == "driver":
+ if isinstance(event, StreamStatus):
+ watchdog.note_status_emitted()
+ else:
+ watchdog.bump()
+ yield event
+ finally:
+ drain_task.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await drain_task
diff --git a/autogpt_platform/backend/backend/copilot/stream_heartbeat_test.py b/autogpt_platform/backend/backend/copilot/stream_heartbeat_test.py
new file mode 100644
index 0000000000..a8754d682b
--- /dev/null
+++ b/autogpt_platform/backend/backend/copilot/stream_heartbeat_test.py
@@ -0,0 +1,315 @@
+"""Unit tests for the silence watchdog.
+
+The watchdog is a small async helper that emits ``StreamStatus`` events
+during long silent gaps in a copilot turn (model deep-thinking before any
+chunk, inter-tool gaps, slow tool execution). Tests here use a tight
+schedule (sub-second thresholds) so they run quickly without flakiness.
+"""
+
+import asyncio
+from collections.abc import AsyncGenerator, AsyncIterator
+
+import pytest
+import pytest_asyncio
+
+from backend.copilot.response_model import (
+ StreamBaseResponse,
+ StreamFinish,
+ StreamStatus,
+ StreamTextDelta,
+)
+from backend.copilot.stream_heartbeat import SilenceWatchdog, wrap_stream_with_heartbeat
+
+
+# Override the heavy session-scope fixtures from ``backend/conftest.py`` —
+# the watchdog is a pure async helper and doesn't need the real backend
+# server / graph cleanup machinery. Mirrors the pattern in
+# ``backend/copilot/sdk/conftest.py``.
+@pytest_asyncio.fixture(scope="session", loop_scope="session", name="server")
+async def _server_noop() -> None:
+ return None
+
+
+@pytest_asyncio.fixture(
+ scope="session", loop_scope="session", autouse=True, name="graph_cleanup"
+)
+async def _graph_cleanup_noop() -> AsyncIterator[None]:
+ yield
+
+
+async def test_emits_status_after_silent_threshold():
+ """When no event is bumped, the watchdog fires the first scheduled
+ status message after its threshold elapses."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [(0.05, "Working on it…")]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ):
+ await asyncio.sleep(0.15)
+
+ assert any(s.message == "Working on it…" for s in received)
+
+
+async def test_bump_resets_silence_timer():
+ """Periodic ``bump()`` calls keep the watchdog quiet — the threshold
+ is measured from the last bump, not from watchdog start."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [(0.10, "Working on it…")]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ) as watchdog:
+ for _ in range(5):
+ await asyncio.sleep(0.04)
+ watchdog.bump()
+ # 0.20s elapsed but each gap was < 0.10s.
+ assert received == []
+
+
+async def test_escalates_through_schedule():
+ """Each threshold fires its own message, in order."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [
+ (0.05, "Working on it…"),
+ (0.10, "Still working…"),
+ (0.20, "This is taking longer…"),
+ ]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ):
+ await asyncio.sleep(0.30)
+
+ messages = [s.message for s in received]
+ assert "Working on it…" in messages
+ assert "Still working…" in messages
+ assert "This is taking longer…" in messages
+ # Order matches the schedule.
+ assert messages.index("Working on it…") < messages.index("Still working…")
+ assert messages.index("Still working…") < messages.index("This is taking longer…")
+
+
+async def test_each_threshold_fires_only_once_per_silent_gap():
+ """A given threshold message fires once per silent gap. After
+ ``bump()`` resets, the schedule re-arms."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [(0.05, "Working on it…")]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ) as watchdog:
+ await asyncio.sleep(0.15)
+ first_count = sum(1 for s in received if s.message == "Working on it…")
+ # Past first threshold, more ticks should not multiply emissions.
+ await asyncio.sleep(0.10)
+ second_count = sum(1 for s in received if s.message == "Working on it…")
+ assert first_count == 1
+ assert second_count == 1
+ # After bump, schedule re-arms.
+ watchdog.bump()
+ await asyncio.sleep(0.10)
+ third_count = sum(1 for s in received if s.message == "Working on it…")
+ assert third_count == 2
+
+
+async def test_suppression_window_skips_when_recent_status():
+ """If a non-watchdog ``StreamStatus`` was emitted recently, the
+ watchdog stays quiet to avoid talking over the driver's own status
+ messages (e.g. 'Optimizing conversation context…')."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [(0.05, "Working on it…")]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.20,
+ ) as watchdog:
+ watchdog.note_status_emitted()
+ await asyncio.sleep(0.15)
+ # Suppression window 0.20s > elapsed 0.15s → quiet.
+ assert received == []
+
+
+async def test_cleanup_on_context_exit():
+ """The internal task is cancelled and awaited when the context
+ manager exits — no orphaned tasks."""
+ received: list[StreamStatus] = []
+
+ async def emit(status: StreamStatus) -> None:
+ received.append(status)
+
+ schedule = [(10.0, "Working on it…")] # never fires in this test
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ ) as watchdog:
+ task = watchdog._task
+ assert task is not None and not task.done()
+
+ assert task.done()
+
+
+async def _delayed_stream(
+ items: list[tuple[float, StreamBaseResponse]],
+) -> AsyncGenerator[StreamBaseResponse, None]:
+ """Helper: yield ``items`` with the given pre-yield sleeps."""
+ for delay, item in items:
+ if delay > 0:
+ await asyncio.sleep(delay)
+ yield item
+
+
+async def test_wrap_passes_through_driver_events_in_order():
+ """The wrapper must not reorder or drop the driver's events."""
+ events = [
+ (0.0, StreamTextDelta(id="t1", delta="a")),
+ (0.0, StreamTextDelta(id="t1", delta="b")),
+ (0.0, StreamFinish()),
+ ]
+ received = []
+ schedule = [(60.0, "Working on it…")] # never fires
+ async for event in wrap_stream_with_heartbeat(
+ _delayed_stream(events),
+ schedule=schedule,
+ tick_s=0.05,
+ ):
+ received.append(event)
+
+ assert len(received) == 3
+ assert received[0].delta == "a" # type: ignore[union-attr]
+ assert received[1].delta == "b" # type: ignore[union-attr]
+ assert isinstance(received[2], StreamFinish)
+
+
+async def test_wrap_emits_status_during_silent_gap():
+ """When the driver pauses, the watchdog interleaves status events."""
+ events = [
+ (0.0, StreamTextDelta(id="t1", delta="hello")),
+ (0.20, StreamFinish()),
+ ]
+ schedule = [(0.05, "Working on it…")]
+ received = []
+ async for event in wrap_stream_with_heartbeat(
+ _delayed_stream(events),
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ):
+ received.append(event)
+
+ statuses = [e for e in received if isinstance(e, StreamStatus)]
+ assert any(s.message == "Working on it…" for s in statuses)
+
+
+async def test_wrap_driver_status_suppresses_watchdog():
+ """A driver-emitted ``StreamStatus`` should keep the watchdog quiet
+ for ``suppression_window_s``. Otherwise both would talk at once."""
+ events = [
+ (0.0, StreamStatus(message="Optimizing conversation context…")),
+ (0.10, StreamFinish()),
+ ]
+ schedule = [(0.05, "Working on it…")]
+ received = []
+ async for event in wrap_stream_with_heartbeat(
+ _delayed_stream(events),
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.20,
+ ):
+ received.append(event)
+
+ watchdog_msgs = [
+ e
+ for e in received
+ if isinstance(e, StreamStatus) and e.message == "Working on it…"
+ ]
+ assert watchdog_msgs == []
+
+
+async def test_emit_status_failure_does_not_kill_watchdog():
+ """If the emit callback raises (e.g. SSE channel went away), the
+ watchdog logs and keeps running so subsequent bumps still work."""
+ call_count = 0
+
+ async def emit(status: StreamStatus) -> None:
+ nonlocal call_count
+ call_count += 1
+ if call_count == 1:
+ raise RuntimeError("emit failed")
+
+ schedule = [(0.05, "Working on it…")]
+ async with SilenceWatchdog(
+ emit_status=emit,
+ schedule=schedule,
+ tick_s=0.01,
+ suppression_window_s=0.0,
+ ) as watchdog:
+ await asyncio.sleep(0.10)
+ # First emit raised — but the watchdog must still be alive.
+ assert watchdog._task is not None
+ assert not watchdog._task.done()
+
+
+async def test_wrap_underlying_stream_closed_on_early_consumer_exit():
+ """Early aclose() on the wrapper must propagate to the underlying stream
+ so its cleanup (e.g. SDK lock release) runs deterministically."""
+ closed = False
+
+ async def _closeable_stream() -> AsyncGenerator[StreamBaseResponse, None]:
+ nonlocal closed
+ try:
+ for _ in range(100):
+ await asyncio.sleep(0.01)
+ yield StreamTextDelta(id="t1", delta="x")
+ finally:
+ closed = True
+
+ schedule = [(60.0, "Working on it…")] # never fires in this test
+ gen = wrap_stream_with_heartbeat(
+ _closeable_stream(), schedule=schedule, tick_s=0.05
+ )
+ await gen.__anext__() # consume one event
+ await gen.aclose() # exit early
+ await asyncio.sleep(0.05) # let cleanup propagate
+ assert closed, "Underlying stream was not closed on early consumer exit"