AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
dx(platform/copilot): reproduce each known cause of the chat drifting from the backend (#15060)
Reinier van der Leere170df9d2ab4
diff --git a/.github/scripts/backend-allowed-skips.json b/.github/scripts/backend-allowed-skips.jsonindex ca0440b1c7..3ff481c369 100644--- a/.github/scripts/backend-allowed-skips.json+++ b/.github/scripts/backend-allowed-skips.json@@ -113,6 +113,12 @@"backend.copilot.sdk.tool_adapter_test.TestBug2FIFODesync.test_old_code_fifo_desync_on_denial","backend.copilot.sdk.tool_adapter_test.TestBug3CancelRace.test_old_code_cancel_arrives_too_late","backend.copilot.service_test.test_sdk_resume_multi_turn",+ "backend.copilot.stream_drift.drift_causes_test.test_a_dispatch_does_not_take_over_a_running_turns_meta",+ "backend.copilot.stream_drift.drift_causes_test.test_a_reply_that_failed_to_persist_ends_the_turn_in_error",+ "backend.copilot.stream_drift.drift_causes_test.test_a_resume_of_a_long_running_turn_replays_whole_blocks",+ "backend.copilot.stream_drift.drift_causes_test.test_a_silent_turn_keeps_publishing_within_the_stream_ttl",+ "backend.copilot.stream_drift.drift_causes_test.test_a_turn_the_executor_drops_is_not_left_running",+ "backend.copilot.stream_drift.drift_causes_test.test_a_turn_whose_executor_died_is_not_reported_running","backend.copilot.test_copilot_e2e.test_session_state_persistence","backend.copilot.tools.agent_browser_integration_test.test_chromium_executable_env_is_set","backend.copilot.tools.agent_browser_integration_test.test_close_session",diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/__init__.py b/autogpt_platform/backend/backend/copilot/stream_drift/__init__.pynew file mode 100644index 0000000000..e69de29bb2diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/conftest.py b/autogpt_platform/backend/backend/copilot/stream_drift/conftest.pynew file mode 100644index 0000000000..ce666e4bfc--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_drift/conftest.py@@ -0,0 +1,59 @@+"""The drift suite needs Redis and nothing else, so it opts out of SpinTestServer."""++from collections.abc import Iterator+from unittest.mock import AsyncMock, MagicMock++import pytest+import pytest_asyncio++from backend.copilot.baseline import service as baseline+from backend.copilot.context import set_execution_context+from backend.copilot.model import ChatSession+from backend.copilot.model_router import ResolvedModel+++@pytest_asyncio.fixture(scope="session", loop_scope="session")+async def server(): # type: ignore[override]+ return None+++@pytest_asyncio.fixture(scope="session", loop_scope="session", autouse=True)+async def graph_cleanup(): # type: ignore[override]+ yield+++def baseline_io(monkeypatch: pytest.MonkeyPatch) -> Iterator[list[ChatSession]]:+ """Every session the baseline engine persists, with its I/O stubbed."""+ persisted: list[ChatSession] = []++ async def save(session: ChatSession) -> ChatSession:+ persisted.append(session.model_copy(deep=True))+ return session++ monkeypatch.setattr(+ baseline,+ "config",+ baseline.config.model_copy(+ update={"use_e2b_sandbox": False, "use_local": False}+ ),+ )+ for name, value in {+ "drain_pending_safe": [],+ "resolve_model_route": ResolvedModel(+ model="anthropic/claude-sonnet-4-6", source="env"+ ),+ "_build_system_prompt": ("System prompt", None),+ "build_turn_budget_block": "",+ "is_enabled_for_user": False,+ "is_feature_enabled": False,+ "persist_and_record_usage": None,+ "download_transcript": None,+ }.items():+ monkeypatch.setattr(baseline, name, AsyncMock(return_value=value))+ monkeypatch.setattr(baseline, "_get_main_client", MagicMock())+ monkeypatch.setattr(baseline, "upsert_chat_session", AsyncMock(side_effect=save))+ try:+ yield persisted+ finally:+ set_execution_context(None, None)diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/drift_causes_test.py b/autogpt_platform/backend/backend/copilot/stream_drift/drift_causes_test.pynew file mode 100644index 0000000000..8e7f5d4c75--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_drift/drift_causes_test.py@@ -0,0 +1,296 @@+"""Backend causes of chat state drifting from server state, via the real registry.++Each live cause is a strict expected failure whose reason names it: CI stays+green while the bug is there, and the test turns red once a fix makes it+pass. ``raises=AssertionError`` keeps a harness error from passing as the bug.+The completion writers' turn guard (#14974) is already pinned by+``stream_registry_test.py::TestCompletionOnRealRedis``.+"""++import asyncio+import functools+import threading+import time+import uuid+from collections.abc import AsyncGenerator+from concurrent.futures import Future+from unittest.mock import AsyncMock, MagicMock, call++import pytest++from backend.copilot import stream_heartbeat, stream_registry+from backend.copilot.baseline import service as baseline+from backend.copilot.executor.manager import CoPilotExecutor+from backend.copilot.executor.utils import CoPilotExecutionEntry, get_session_lock_key+from backend.copilot.model import ChatSession+from backend.copilot.response_model import (+ StreamBaseResponse,+ StreamError,+ StreamReasoningDelta,+ StreamReasoningEnd,+ StreamReasoningStart,+ StreamStart,+ StreamStatus,+ StreamTextDelta,+ StreamTextEnd,+ StreamTextStart,+)+from backend.data import redis_client+from backend.data.redis_client import get_redis_async+from backend.util.testing import is_tcp_port_reachable++from .scripted import baseline_turn, provider_round, session_with_prompt++requires_redis = pytest.mark.skipif(+ not is_tcp_port_reachable(redis_client.HOST, redis_client.PORT),+ reason="no local Redis reachable; the stream registry needs one to run",+)+++async def session_id():+ session_id = f"drift-{uuid.uuid4().hex}"+ yield session_id+ redis = await get_redis_async()+ await redis.delete(stream_registry.get_session_meta_key(session_id))+ await redis.delete(get_session_lock_key(session_id))+++async def turn_id():+ turn_id = str(uuid.uuid4())+ yield turn_id+ redis = await get_redis_async()+ await redis.delete(stream_registry._get_turn_stream_key(turn_id))+++@requires_redis+ strict=True,+ raises=AssertionError,+ reason=(+ "W2: create_session overwrites the meta of a turn that is still "+ "running, so every reader follows the new turn and the running one "+ "streams to nobody"+ ),+)+async def test_a_dispatch_does_not_take_over_a_running_turns_meta(+ session_id: str,+) -> None:+ await stream_registry.create_session(session_id, None, "", "", turn_id="a")+ # A second POST let through acquire_turn_slot's refresh branch.+ await stream_registry.create_session(session_id, None, "", "", turn_id="b")++ active, _ = await stream_registry.get_active_session(session_id)+ assert active is not None and active.turn_id == "a"+++@requires_redis+ strict=True,+ raises=AssertionError,+ reason=(+ "W3: a turn whose executor died reads as running until its meta "+ "expires, up to an hour; the lapsed cluster lock is never consulted"+ ),+)+async def test_a_turn_whose_executor_died_is_not_reported_running(+ session_id: str, turn_id: str+) -> None:+ await stream_registry.create_session(session_id, None, "", "", turn_id=turn_id)+ # The executor publishes this only after taking the cluster lock.+ await stream_registry.publish_chunk(+ turn_id,+ StreamStatus(message="Setting up your environment…"),+ session_id=session_id,+ )+ # Its pod is gone: nothing refreshes the lock and it has lapsed.+ redis = await get_redis_async()+ if await redis.exists(get_session_lock_key(session_id)):+ pytest.fail("harness: the cluster lock should be gone")++ active, _ = await stream_registry.get_active_session(session_id)+ assert active is None+++@requires_redis+ strict=True,+ raises=AssertionError,+ reason=(+ "W5: publish_chunk trims a running turn's stream to stream_max_length, "+ "so a resume replays from mid-block and the client's parser throws"+ ),+)+async def test_a_resume_of_a_long_running_turn_replays_whole_blocks(+ monkeypatch: pytest.MonkeyPatch, session_id: str, turn_id: str+) -> None:+ monkeypatch.setattr(+ stream_registry,+ "config",+ stream_registry.config.model_copy(update={"stream_max_length": 20}),+ )+ await stream_registry.create_session(session_id, None, "", "", turn_id=turn_id)+ for chunk in [+ StreamStart(messageId="m1"),+ StreamTextStart(id="t1"),+ *(StreamTextDelta(id="t1", delta=f"{i} ") for i in range(300)),+ ]:+ await stream_registry.publish_chunk(turn_id, chunk, session_id=session_id)++ queue = await stream_registry.subscribe_to_session(session_id, None, "0-0")+ if queue is None:+ pytest.fail("harness: the running turn has no subscribable stream")+ try:+ replay = [queue.get_nowait() for _ in range(queue.qsize())]+ finally:+ await stream_registry.unsubscribe_from_session(session_id, queue)++ assert _orphan_parts(replay) == []+++ strict=True,+ raises=AssertionError,+ reason=(+ "W6: after the silence watchdog's last tier a silent turn publishes "+ "nothing, so nothing refreshes the session meta's TTL and it expires "+ "under a live turn"+ ),+)+async def test_a_silent_turn_keeps_publishing_within_the_stream_ttl(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ ttl = stream_registry.config.stream_ttl+ now = 0.0++ def clock() -> float:+ nonlocal now+ now += 5.0+ return now++ monkeypatch.setattr(+ stream_heartbeat,+ "SilenceWatchdog",+ functools.partial(stream_heartbeat.SilenceWatchdog, clock=clock),+ )+ ended = asyncio.Event()++ async def silent_engine() -> AsyncGenerator[StreamBaseResponse, None]:+ yield StreamStart(messageId="m1")+ await ended.wait()++ published_at: list[float] = []++ async def consume() -> None:+ async for _ in stream_heartbeat.wrap_stream_with_heartbeat(+ silent_engine(), tick_s=0.001+ ):+ published_at.append(now)++ consumer = asyncio.create_task(consume())+ try:+ async with asyncio.timeout(30):+ while now < 2 * ttl:+ await asyncio.sleep(0.01)+ ended.set()+ await consumer+ finally:+ ended.set()+ consumer.cancel()++ gaps = [b - a for a, b in zip(published_at, [*published_at[1:], now])]+ assert max(gaps) < ttl+++@requires_redis+ strict=True,+ raises=AssertionError,+ reason=(+ "Approval wake: the end-of-turn wake dispatches the follow-up turn while "+ "the ending turn is still registered, and the executor drops it as a "+ "duplicate without closing it, so the chat reads running forever"+ ),+)+async def test_a_turn_the_executor_drops_is_not_left_running(+ session_id: str, turn_id: str+) -> None:+ # held.wake -> dispatch_turn writes the follow-up turn's meta, then enqueues it.+ await stream_registry.create_session(session_id, None, "", "", turn_id=turn_id)+ executor = CoPilotExecutor()+ # The turn whose end woke it has not left the executor yet.+ executor.active_tasks[session_id] = (Future(), threading.Event())+ channel = MagicMock(is_open=True)+ channel.connection.add_callback_threadsafe.side_effect = lambda ack: ack()+ entry = CoPilotExecutionEntry(+ session_id=session_id, turn_id=turn_id, user_id=None, message="wake"+ )++ executor._handle_run_message(+ channel,+ MagicMock(delivery_tag=1),+ MagicMock(),+ entry.model_dump_json().encode(),+ )++ dropped = channel.basic_nack.call_args == call(1, requeue=False)+ # A fix may close the dropped turn asynchronously.+ deadline = time.monotonic() + 5+ while dropped and time.monotonic() < deadline:+ if not await _reads_running(session_id):+ break+ await asyncio.sleep(0.1)+ assert not (+ dropped and await _reads_running(session_id)+ ), "the executor dropped the turn and it still reads running"+++ strict=True,+ raises=AssertionError,+ reason=(+ "W8: the baseline engine logs and swallows a failed final persist, so "+ "the turn finishes as a success and the streamed reply is not in the "+ "rows the client hydrates from"+ ),+)+async def test_a_reply_that_failed_to_persist_ends_the_turn_in_error(+ monkeypatch: pytest.MonkeyPatch, baseline_io: list[ChatSession]+) -> None:+ monkeypatch.setattr(+ baseline,+ "call_provider_stream",+ AsyncMock(return_value=provider_round(["The answer ", "is 42."])),+ )+ persist = AsyncMock(side_effect=ConnectionError("database unavailable"))+ monkeypatch.setattr(baseline, "upsert_chat_session", persist)++ turn = baseline_turn(session_with_prompt("Why?"), str(uuid.uuid4()))+ events = [event async for event in turn]++ streamed = "".join(e.delta for e in events if isinstance(e, StreamTextDelta))+ if streamed != "The answer is 42." or not persist.await_count:+ pytest.fail("harness: the reply should stream and its persist should run")+ assert any(isinstance(event, StreamError) for event in events)+++async def _reads_running(session_id: str) -> bool:+ active, _ = await stream_registry.get_active_session(session_id)+ return active is not None+++def _orphan_parts(chunks: list[StreamBaseResponse]) -> list[str]:+ """Deltas and ends whose start the replay does not contain."""+ open_ids: set[str] = set()+ orphans = []+ for chunk in chunks:+ if isinstance(chunk, (StreamTextStart, StreamReasoningStart)):+ open_ids.add(chunk.id)+ elif isinstance(+ chunk,+ (StreamTextDelta, StreamTextEnd, StreamReasoningDelta, StreamReasoningEnd),+ ):+ if chunk.id not in open_ids:+ orphans.append(f"{chunk.type.value}:{chunk.id}")+ return orphansdiff --git a/autogpt_platform/backend/backend/copilot/stream_drift/recorded_turns_test.py b/autogpt_platform/backend/backend/copilot/stream_drift/recorded_turns_test.pynew file mode 100644index 0000000000..ab80246ab0--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_drift/recorded_turns_test.py@@ -0,0 +1,133 @@+"""The recorded turns the frontend drift suite replays still match the pipeline.++Each test drives an engine through the real registry and Redis, then compares+what it stored, served and persisted with the committed fixture. A red here+means the sender changed; re-record and rerun the frontend suite.+"""++import uuid+from unittest.mock import AsyncMock++import pytest+from claude_agent_sdk import (+ AssistantMessage,+ ResultMessage,+ SystemMessage,+ TextBlock,+ ToolResultBlock,+ ToolUseBlock,+ UserMessage,+)++from backend.copilot.baseline import service as baseline+from backend.copilot.model import ChatSession+from backend.copilot.response_model import StreamToolOutputAvailable+from backend.copilot.sdk import dummy+from backend.data import redis_client+from backend.util.testing import is_tcp_port_reachable++from .recording import check_fixture, record_turn+from .scripted import baseline_turn, provider_round, sdk_turn, session_with_prompt++pytestmark = pytest.mark.skipif(+ not is_tcp_port_reachable(redis_client.HOST, redis_client.PORT),+ reason="no local Redis reachable; the stream registry needs one to run",+)+++async def test_dummy_text_turn(monkeypatch: pytest.MonkeyPatch) -> None:+ session = session_with_prompt("Count to three")+ persisted: list[ChatSession] = []++ async def save(saved: ChatSession) -> ChatSession:+ persisted.append(saved.model_copy(deep=True))+ return saved++ monkeypatch.setattr(dummy, "upsert_chat_session", save)++ recorded = await record_turn(+ dummy.stream_chat_completion_dummy(+ session.session_id, message="Count to three", session=session+ ),+ session_id=session.session_id,+ turn_id=str(uuid.uuid4()),+ persisted=lambda: persisted[-1].messages,+ )++ check_fixture("dummy-text-turn", recorded)+++async def test_baseline_tool_turn(+ monkeypatch: pytest.MonkeyPatch, baseline_io: list[ChatSession]+) -> None:+ session = session_with_prompt("What does example.com say?")+ rounds = [+ provider_round(+ ["Let me ", "fetch that ", "page."],+ tool_call={"id": "call-fetch", "name": "web_fetch", "url": "example.com"},+ ),+ provider_round(["It says ", "Example Domain."]),+ ]+ monkeypatch.setattr(baseline, "call_provider_stream", AsyncMock(side_effect=rounds))++ async def execute(*, tool_call_id: str, **_: object) -> StreamToolOutputAvailable:+ return StreamToolOutputAvailable(+ toolCallId=tool_call_id,+ toolName="web_fetch",+ output="<h1>Example Domain</h1>",+ )++ monkeypatch.setattr(baseline, "execute_tool", AsyncMock(side_effect=execute))+ turn_id = str(uuid.uuid4())++ recorded = await record_turn(+ baseline_turn(session, turn_id),+ session_id=session.session_id,+ turn_id=turn_id,+ persisted=lambda: baseline_io[-1].messages,+ )++ check_fixture("baseline-tool-turn", recorded)+++async def test_sdk_late_tool_result_turn() -> None:+ """Dev session c68992f9's order: a tool result lands after the next text."""+ session = session_with_prompt("Check the task and list the files")+ messages = [+ SystemMessage(subtype="init", data={}),+ AssistantMessage(+ content=[ToolUseBlock(id="task-1", name="TaskOutput", input={})],+ model="drift",+ ),+ AssistantMessage(content=[TextBlock(text="The task finished.")], model="drift"),+ UserMessage(content=[ToolResultBlock(tool_use_id="task-1", content="late")]),+ AssistantMessage(+ content=[+ ToolUseBlock(id="bash-1", name="bash_exec", input={"command": "ls"})+ ],+ model="drift",+ ),+ UserMessage(+ content=[ToolResultBlock(tool_use_id="bash-1", content="report.md")]+ ),+ AssistantMessage(+ content=[TextBlock(text="All done: report.md is ready.")], model="drift"+ ),+ ResultMessage(+ subtype="success",+ duration_ms=100,+ duration_api_ms=50,+ is_error=False,+ num_turns=3,+ session_id="drift",+ ),+ ]++ recorded = await record_turn(+ sdk_turn(session, messages),+ session_id=session.session_id,+ turn_id=str(uuid.uuid4()),+ persisted=lambda: session.messages,+ )++ check_fixture("sdk-late-tool-result", recorded)diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/recording.py b/autogpt_platform/backend/backend/copilot/stream_drift/recording.pynew file mode 100644index 0000000000..ccbe9ece67--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_drift/recording.py@@ -0,0 +1,188 @@+"""Record a copilot turn the way production stores it, serves it and persists it.++A recorded turn is three files under ``test/fixtures/copilot_stream/<name>/``:++- ``entries.jsonl``: the turn's Redis stream in XRANGE order, ``{id, data}``+ per line with ``data`` parsed from the stored JSON;+- ``frames.jsonl``: the SSE frame the resume route writes for each entry;+- ``rows.json``: the rows the session GET returns once the turn persisted.++Backend tests check the pipeline still records these files; the frontend drift+suite replays them. Set ``RECORD_COPILOT_STREAM_FIXTURES=1`` to rewrite them.+"""++import json+import os+import re+from collections.abc import AsyncGenerator, Callable, Sequence+from pathlib import Path+from typing import Any+from unittest.mock import AsyncMock, patch++import orjson+from fastapi.encoders import jsonable_encoder+from pydantic import BaseModel++from backend.copilot import stream_registry+from backend.copilot.model import ChatMessage+from backend.copilot.response_model import StreamBaseResponse, StreamError, StreamStatus+from backend.copilot.stream_heartbeat import wrap_stream_with_heartbeat+from backend.data.redis_client import get_redis_async++FIXTURE_ROOT = (+ Path(__file__).resolve().parents[3] / "test" / "fixtures" / "copilot_stream"+)+RECORD_ENV = "RECORD_COPILOT_STREAM_FIXTURES"++CANONICAL_SESSION_ID = "drift-session"+CANONICAL_TURN_ID = "drift-turn"++_UUID = re.compile(r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}")+++class RecordedTurn(BaseModel):+ entries: list[dict[str, Any]]+ frames: list[dict[str, Any]]+ rows: list[dict[str, Any]]+++async def record_turn(+ engine: AsyncGenerator[StreamBaseResponse, None],+ *,+ session_id: str,+ turn_id: str,+ persisted: Callable[[], Sequence[ChatMessage]],+) -> RecordedTurn:+ """Run ``engine`` through the route's and the executor's publishing path."""+ await stream_registry.create_session(session_id, None, "", "", turn_id=turn_id)+ for status in ("Message received…", "Setting up your environment…"):+ await stream_registry.publish_chunk(+ turn_id, StreamStatus(message=status), session_id=session_id+ )++ error: str | None = None+ published = stream_registry.stream_and_publish(+ session_id=session_id,+ turn_id=turn_id,+ stream=wrap_stream_with_heartbeat(engine),+ )+ try:+ async for chunk in published:+ if isinstance(chunk, StreamError):+ error = chunk.errorText+ break+ finally:+ await published.aclose()++ with patch.object(+ stream_registry.chat_db(), "set_turn_duration", new=AsyncMock(), create=True+ ):+ await stream_registry.mark_session_completed(+ session_id, error_message=error, turn_id=turn_id+ )+ entries = await read_turn_entries(turn_id)+ redis = await get_redis_async()+ await redis.delete(stream_registry._get_turn_stream_key(turn_id))+ await redis.delete(stream_registry.get_session_meta_key(session_id))+ return canonical(+ RecordedTurn(+ entries=entries,+ frames=frames_for(entries),+ rows=[jsonable_encoder(message.model_dump()) for message in persisted()],+ ),+ session_id=session_id,+ turn_id=turn_id,+ )+++async def read_turn_entries(turn_id: str) -> list[dict[str, Any]]:+ redis = await get_redis_async()+ key = stream_registry._get_turn_stream_key(turn_id)+ [(_, entries)] = stream_registry._stream_entries([(key, await redis.xrange(key))])+ return [+ {"id": entry_id, "data": orjson.loads(fields["data"])}+ for entry_id, fields in entries+ ]+++def frames_for(entries: Sequence[dict[str, Any]]) -> list[dict[str, Any]]:+ """The frames the resume route writes, one per entry it can rebuild."""+ frames = []+ for entry in entries:+ chunk = stream_registry._reconstruct_chunk(entry["data"])+ if chunk is not None:+ frames.append({"id": entry["id"], "sse": chunk.to_sse()})+ return frames+++def canonical(turn: RecordedTurn, *, session_id: str, turn_id: str) -> RecordedTurn:+ """Replace every value that differs between two recordings of one turn."""+ entry_ids = {entry["id"]: f"{i + 1}-0" for i, entry in enumerate(turn.entries)}+ rows = [+ {+ **row,+ "id": f"row-{sequence}",+ "sequence": sequence,+ "created_at": f"2026-09-30T00:00:{sequence:02d}Z",+ }+ for sequence, row in enumerate(turn.rows)+ ]+ text = json.dumps(+ {+ "entries": [{**e, "id": entry_ids[e["id"]]} for e in turn.entries],+ "frames": [{**f, "id": entry_ids[f["id"]]} for f in turn.frames],+ "rows": rows,+ }+ )+ text = text.replace(session_id, CANONICAL_SESSION_ID)+ text = text.replace(turn_id, CANONICAL_TURN_ID)+ uuids: dict[str, str] = {}+ text = _UUID.sub(+ lambda m: uuids.setdefault(+ m.group(0), f"00000000-0000-4000-8000-{len(uuids) + 1:012d}"+ ),+ text,+ )+ return RecordedTurn.model_validate_json(text)+++def load_fixture(name: str) -> RecordedTurn:+ directory = FIXTURE_ROOT / name+ return RecordedTurn(+ entries=_read_jsonl(directory / "entries.jsonl"),+ frames=_read_jsonl(directory / "frames.jsonl"),+ rows=json.loads((directory / "rows.json").read_text()),+ )+++def check_fixture(name: str, recorded: RecordedTurn) -> None:+ """Fail when the pipeline no longer records the committed fixture."""+ if os.environ.get(RECORD_ENV):+ _write_fixture(name, recorded)+ return+ committed = load_fixture(name)+ assert recorded == committed, (+ f"The pipeline no longer records {FIXTURE_ROOT / name}. If the change is "+ f"intended, re-record with {RECORD_ENV}=1 and check the frontend drift "+ "suite still passes on the new files."+ )+++def _write_fixture(name: str, turn: RecordedTurn) -> None:+ directory = FIXTURE_ROOT / name+ directory.mkdir(parents=True, exist_ok=True)+ _write_jsonl(directory / "entries.jsonl", turn.entries)+ _write_jsonl(directory / "frames.jsonl", turn.frames)+ (directory / "rows.json").write_text(+ json.dumps(turn.rows, indent=2, ensure_ascii=False) + "\n"+ )+++def _read_jsonl(path: Path) -> list[dict[str, Any]]:+ return [json.loads(line) for line in path.read_text().splitlines() if line]+++def _write_jsonl(path: Path, items: Sequence[dict[str, Any]]) -> None:+ path.write_text(+ "".join(json.dumps(item, ensure_ascii=False) + "\n" for item in items)+ )diff --git a/autogpt_platform/backend/backend/copilot/stream_drift/scripted.py b/autogpt_platform/backend/backend/copilot/stream_drift/scripted.pynew file mode 100644index 0000000000..97cf520df1--- /dev/null+++ b/autogpt_platform/backend/backend/copilot/stream_drift/scripted.py@@ -0,0 +1,115 @@+"""Copilot engines driven by a script instead of a model, for the drift suite."""++import uuid+from collections.abc import AsyncGenerator+from typing import Any+from unittest.mock import AsyncMock, MagicMock++from claude_agent_sdk import Message+from openai.types.chat import ChatCompletionChunk++from backend.copilot.baseline import service as baseline+from backend.copilot.model import ChatMessage, ChatSession+from backend.copilot.response_model import StreamBaseResponse, StreamStart+from backend.copilot.sdk.response_adapter import SDKResponseAdapter+from backend.copilot.sdk.service import _dispatch_response, _StreamAccumulator+from backend.copilot.tree import root_envelope+++def session_with_prompt(prompt: str) -> ChatSession:+ session = ChatSession.new("drift-user", dry_run=False)+ session.title = "Drift fixture"+ session.messages.append(ChatMessage(role="user", content=prompt))+ return session+++async def baseline_turn(+ session: ChatSession, turn_id: str+) -> AsyncGenerator[StreamBaseResponse, None]:+ """The baseline engine; its provider and I/O come from the ``baseline_io`` fixture."""+ async for event in baseline.stream_chat_completion_baseline(+ session.session_id,+ user_id="drift-user",+ session=session,+ is_user_message=False,+ envelope=root_envelope(turn_id, session_id=session.session_id),+ ):+ yield event+++def provider_round(+ deltas: list[str], *, tool_call: dict[str, str] | None = None+) -> MagicMock:+ """One scripted OpenAI-compatible response: text deltas, then a tool call."""+ chunks = [_chunk({"content": delta}) for delta in deltas]+ if tool_call:+ chunks.append(+ _chunk(+ {+ "tool_calls": [+ {+ "index": 0,+ "id": tool_call["id"],+ "type": "function",+ "function": {+ "name": tool_call["name"],+ "arguments": f'{{"url": "{tool_call["url"]}"}}',+ },+ }+ ]+ }+ )+ )+ chunks.append(+ _chunk({}, finish_reason="tool_calls" if tool_call else "stop", usage=True)+ )+ stream = MagicMock()+ stream.__aiter__.return_value = chunks+ stream.close = AsyncMock()+ return stream+++async def sdk_turn(+ session: ChatSession, messages: list[Message]+) -> AsyncGenerator[StreamBaseResponse, None]:+ """The SDK engine's own adapter and row builder over a scripted CLI."""+ message_id = str(uuid.uuid4())+ adapter = SDKResponseAdapter(message_id=message_id, session_id=session.session_id)+ acc = _StreamAccumulator(+ assistant_response=ChatMessage(role="assistant", content=""),+ accumulated_tool_calls=[],+ )+ ctx = MagicMock(session=session, log_prefix="[drift]")+ yield StreamStart(messageId=message_id, sessionId=session.session_id)+ for message in messages:+ for response in adapter.convert_message(message):+ dispatched = _dispatch_response(+ response, acc, ctx, MagicMock(), False, "[drift]"+ )+ if dispatched is not None:+ yield dispatched+++def _chunk(+ delta: dict[str, Any], *, finish_reason: str | None = None, usage: bool = False+) -> ChatCompletionChunk:+ return ChatCompletionChunk.model_validate(+ {+ "id": "round",+ "object": "chat.completion.chunk",+ "created": 0,+ "model": "anthropic/claude-sonnet-4-6",+ "choices": [{"index": 0, "finish_reason": finish_reason, "delta": delta}],+ **(+ {+ "usage": {+ "prompt_tokens": 100,+ "completion_tokens": 20,+ "total_tokens": 120,+ }+ }+ if usage+ else {}+ ),+ }+ )diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/entries.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/entries.jsonlnew file mode 100644index 0000000000..1b3d1a3733--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/entries.jsonl@@ -0,0 +1,21 @@+{"id": "1-0", "data": {"type": "data-status", "message": "Message received…"}}+{"id": "2-0", "data": {"type": "data-status", "message": "Setting up your environment…"}}+{"id": "3-0", "data": {"type": "start", "messageId": "00000000-0000-4000-8000-000000000001", "sessionId": "drift-session"}}+{"id": "4-0", "data": {"type": "start-step"}}+{"id": "5-0", "data": {"type": "text-start", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "6-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "Let me "}}+{"id": "7-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "fetch that "}}+{"id": "8-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "page."}}+{"id": "9-0", "data": {"type": "text-end", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "10-0", "data": {"type": "finish-step"}}+{"id": "11-0", "data": {"type": "tool-input-start", "toolCallId": "call-fetch", "toolName": "web_fetch"}}+{"id": "12-0", "data": {"type": "tool-input-available", "toolCallId": "call-fetch", "toolName": "web_fetch", "input": {"url": "example.com"}}}+{"id": "13-0", "data": {"type": "tool-output-available", "toolCallId": "call-fetch", "output": "<h1>Example Domain</h1>", "toolName": "web_fetch", "success": true}}+{"id": "14-0", "data": {"type": "start-step"}}+{"id": "15-0", "data": {"type": "text-start", "id": "00000000-0000-4000-8000-000000000003"}}+{"id": "16-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000003", "delta": "It says "}}+{"id": "17-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000003", "delta": "Example Domain."}}+{"id": "18-0", "data": {"type": "text-end", "id": "00000000-0000-4000-8000-000000000003"}}+{"id": "19-0", "data": {"type": "finish-step"}}+{"id": "20-0", "data": {"type": "usage", "prompt_tokens": 200, "completion_tokens": 40, "total_tokens": 240, "cache_read_tokens": 0, "cache_creation_tokens": 0}}+{"id": "21-0", "data": {"type": "finish"}}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/frames.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/frames.jsonlnew file mode 100644index 0000000000..559c50c55f--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/frames.jsonl@@ -0,0 +1,21 @@+{"id": "1-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Message received\\u2026\"}}\n\n"}+{"id": "2-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Setting up your environment\\u2026\"}}\n\n"}+{"id": "3-0", "sse": "data: {\"type\": \"start\", \"messageId\": \"00000000-0000-4000-8000-000000000001\"}\n\n"}+{"id": "4-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "5-0", "sse": "data: {\"type\":\"text-start\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "6-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"Let me \"}\n\n"}+{"id": "7-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"fetch that \"}\n\n"}+{"id": "8-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"page.\"}\n\n"}+{"id": "9-0", "sse": "data: {\"type\":\"text-end\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "10-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "11-0", "sse": "data: {\"type\":\"tool-input-start\",\"toolCallId\":\"call-fetch\",\"toolName\":\"web_fetch\"}\n\n"}+{"id": "12-0", "sse": "data: {\"type\":\"tool-input-available\",\"toolCallId\":\"call-fetch\",\"toolName\":\"web_fetch\",\"input\":{\"url\":\"example.com\"}}\n\n"}+{"id": "13-0", "sse": "data: {\"type\": \"tool-output-available\", \"toolCallId\": \"call-fetch\", \"output\": \"<h1>Example Domain</h1>\"}\n\n"}+{"id": "14-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "15-0", "sse": "data: {\"type\":\"text-start\",\"id\":\"00000000-0000-4000-8000-000000000003\"}\n\n"}+{"id": "16-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000003\",\"delta\":\"It says \"}\n\n"}+{"id": "17-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000003\",\"delta\":\"Example Domain.\"}\n\n"}+{"id": "18-0", "sse": "data: {\"type\":\"text-end\",\"id\":\"00000000-0000-4000-8000-000000000003\"}\n\n"}+{"id": "19-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "20-0", "sse": ": usage {\"type\":\"usage\",\"promptTokens\":200,\"completionTokens\":40,\"totalTokens\":240,\"cacheReadTokens\":0,\"cacheCreationTokens\":0}\n\n"}+{"id": "21-0", "sse": "data: {\"type\":\"finish\"}\n\n"}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/rows.json b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/rows.jsonnew file mode 100644index 0000000000..b2212ee6a0--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/baseline-tool-turn/rows.json@@ -0,0 +1,83 @@+[+ {+ "id": "row-0",+ "role": "user",+ "content": "What does example.com say?",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 0,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:00Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-1",+ "role": "assistant",+ "content": "Let me fetch that page.",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": [+ {+ "id": "call-fetch",+ "type": "function",+ "function": {+ "name": "web_fetch",+ "arguments": "{\"url\": \"example.com\"}"+ }+ }+ ],+ "function_call": null,+ "sequence": 1,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:01Z",+ "model": "anthropic/claude-sonnet-4-6",+ "llm_auth_provider": "platform",+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-2",+ "role": "tool",+ "content": "<h1>Example Domain</h1>",+ "name": null,+ "tool_call_id": "call-fetch",+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 2,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:02Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-3",+ "role": "assistant",+ "content": "It says Example Domain.",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 3,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:03Z",+ "model": "anthropic/claude-sonnet-4-6",+ "llm_auth_provider": "platform",+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ }+]diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/entries.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/entries.jsonlnew file mode 100644index 0000000000..4b07d73dc2--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/entries.jsonl@@ -0,0 +1,15 @@+{"id": "1-0", "data": {"type": "data-status", "message": "Message received…"}}+{"id": "2-0", "data": {"type": "data-status", "message": "Setting up your environment…"}}+{"id": "3-0", "data": {"type": "start", "messageId": "00000000-0000-4000-8000-000000000001", "sessionId": "drift-session"}}+{"id": "4-0", "data": {"type": "start-step"}}+{"id": "5-0", "data": {"type": "text-start", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "6-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "I "}}+{"id": "7-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "counted: "}}+{"id": "8-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "1... "}}+{"id": "9-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "2... "}}+{"id": "10-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "3. "}}+{"id": "11-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "All "}}+{"id": "12-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "done!"}}+{"id": "13-0", "data": {"type": "text-end", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "14-0", "data": {"type": "finish-step"}}+{"id": "15-0", "data": {"type": "finish"}}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/frames.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/frames.jsonlnew file mode 100644index 0000000000..0d3da38e56--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/frames.jsonl@@ -0,0 +1,15 @@+{"id": "1-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Message received\\u2026\"}}\n\n"}+{"id": "2-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Setting up your environment\\u2026\"}}\n\n"}+{"id": "3-0", "sse": "data: {\"type\": \"start\", \"messageId\": \"00000000-0000-4000-8000-000000000001\"}\n\n"}+{"id": "4-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "5-0", "sse": "data: {\"type\":\"text-start\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "6-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"I \"}\n\n"}+{"id": "7-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"counted: \"}\n\n"}+{"id": "8-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"1... \"}\n\n"}+{"id": "9-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"2... \"}\n\n"}+{"id": "10-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"3. \"}\n\n"}+{"id": "11-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"All \"}\n\n"}+{"id": "12-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"done!\"}\n\n"}+{"id": "13-0", "sse": "data: {\"type\":\"text-end\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "14-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "15-0", "sse": "data: {\"type\":\"finish\"}\n\n"}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/rows.json b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/rows.jsonnew file mode 100644index 0000000000..52d81cbe04--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/dummy-text-turn/rows.json@@ -0,0 +1,38 @@+[+ {+ "id": "row-0",+ "role": "user",+ "content": "Count to three",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 0,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:00Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-1",+ "role": "assistant",+ "content": "I counted: 1... 2... 3. All done!",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 1,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:01Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ }+]diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/entries.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/entries.jsonlnew file mode 100644index 0000000000..8d8b6d23b1--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/entries.jsonl@@ -0,0 +1,25 @@+{"id": "1-0", "data": {"type": "data-status", "message": "Message received…"}}+{"id": "2-0", "data": {"type": "data-status", "message": "Setting up your environment…"}}+{"id": "3-0", "data": {"type": "start", "messageId": "00000000-0000-4000-8000-000000000001", "sessionId": "drift-session"}}+{"id": "4-0", "data": {"type": "start-step"}}+{"id": "5-0", "data": {"type": "tool-input-start", "toolCallId": "task-1", "toolName": "TaskOutput"}}+{"id": "6-0", "data": {"type": "tool-input-available", "toolCallId": "task-1", "toolName": "TaskOutput", "input": {}}}+{"id": "7-0", "data": {"type": "tool-output-available", "toolCallId": "task-1", "output": "", "toolName": "TaskOutput", "success": true}}+{"id": "8-0", "data": {"type": "finish-step"}}+{"id": "9-0", "data": {"type": "start-step"}}+{"id": "10-0", "data": {"type": "text-start", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "11-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000002", "delta": "The task finished."}}+{"id": "12-0", "data": {"type": "text-end", "id": "00000000-0000-4000-8000-000000000002"}}+{"id": "13-0", "data": {"type": "finish-step"}}+{"id": "14-0", "data": {"type": "start-step"}}+{"id": "15-0", "data": {"type": "tool-input-start", "toolCallId": "bash-1", "toolName": "bash_exec"}}+{"id": "16-0", "data": {"type": "tool-input-available", "toolCallId": "bash-1", "toolName": "bash_exec", "input": {"command": "ls"}}}+{"id": "17-0", "data": {"type": "tool-output-available", "toolCallId": "bash-1", "output": "report.md", "toolName": "bash_exec", "success": true}}+{"id": "18-0", "data": {"type": "finish-step"}}+{"id": "19-0", "data": {"type": "data-status", "message": "Analyzing result…"}}+{"id": "20-0", "data": {"type": "start-step"}}+{"id": "21-0", "data": {"type": "text-start", "id": "00000000-0000-4000-8000-000000000003"}}+{"id": "22-0", "data": {"type": "text-delta", "id": "00000000-0000-4000-8000-000000000003", "delta": "All done: report.md is ready."}}+{"id": "23-0", "data": {"type": "text-end", "id": "00000000-0000-4000-8000-000000000003"}}+{"id": "24-0", "data": {"type": "finish-step"}}+{"id": "25-0", "data": {"type": "finish"}}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/frames.jsonl b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/frames.jsonlnew file mode 100644index 0000000000..7eb60c038d--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/frames.jsonl@@ -0,0 +1,25 @@+{"id": "1-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Message received\\u2026\"}}\n\n"}+{"id": "2-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Setting up your environment\\u2026\"}}\n\n"}+{"id": "3-0", "sse": "data: {\"type\": \"start\", \"messageId\": \"00000000-0000-4000-8000-000000000001\"}\n\n"}+{"id": "4-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "5-0", "sse": "data: {\"type\":\"tool-input-start\",\"toolCallId\":\"task-1\",\"toolName\":\"TaskOutput\"}\n\n"}+{"id": "6-0", "sse": "data: {\"type\":\"tool-input-available\",\"toolCallId\":\"task-1\",\"toolName\":\"TaskOutput\",\"input\":{}}\n\n"}+{"id": "7-0", "sse": "data: {\"type\": \"tool-output-available\", \"toolCallId\": \"task-1\", \"output\": \"\"}\n\n"}+{"id": "8-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "9-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "10-0", "sse": "data: {\"type\":\"text-start\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "11-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000002\",\"delta\":\"The task finished.\"}\n\n"}+{"id": "12-0", "sse": "data: {\"type\":\"text-end\",\"id\":\"00000000-0000-4000-8000-000000000002\"}\n\n"}+{"id": "13-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "14-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "15-0", "sse": "data: {\"type\":\"tool-input-start\",\"toolCallId\":\"bash-1\",\"toolName\":\"bash_exec\"}\n\n"}+{"id": "16-0", "sse": "data: {\"type\":\"tool-input-available\",\"toolCallId\":\"bash-1\",\"toolName\":\"bash_exec\",\"input\":{\"command\":\"ls\"}}\n\n"}+{"id": "17-0", "sse": "data: {\"type\": \"tool-output-available\", \"toolCallId\": \"bash-1\", \"output\": \"report.md\"}\n\n"}+{"id": "18-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "19-0", "sse": "data: {\"type\": \"data-status\", \"data\": {\"message\": \"Analyzing result\\u2026\"}}\n\n"}+{"id": "20-0", "sse": "data: {\"type\":\"start-step\"}\n\n"}+{"id": "21-0", "sse": "data: {\"type\":\"text-start\",\"id\":\"00000000-0000-4000-8000-000000000003\"}\n\n"}+{"id": "22-0", "sse": "data: {\"type\":\"text-delta\",\"id\":\"00000000-0000-4000-8000-000000000003\",\"delta\":\"All done: report.md is ready.\"}\n\n"}+{"id": "23-0", "sse": "data: {\"type\":\"text-end\",\"id\":\"00000000-0000-4000-8000-000000000003\"}\n\n"}+{"id": "24-0", "sse": "data: {\"type\":\"finish-step\"}\n\n"}+{"id": "25-0", "sse": "data: {\"type\":\"finish\"}\n\n"}diff --git a/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/rows.json b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/rows.jsonnew file mode 100644index 0000000000..d2b9cb747f--- /dev/null+++ b/autogpt_platform/backend/test/fixtures/copilot_stream/sdk-late-tool-result/rows.json@@ -0,0 +1,128 @@+[+ {+ "id": "row-0",+ "role": "user",+ "content": "Check the task and list the files",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 0,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:00Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-1",+ "role": "assistant",+ "content": "",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": [+ {+ "id": "task-1",+ "type": "function",+ "function": {+ "name": "TaskOutput",+ "arguments": "{}"+ }+ }+ ],+ "function_call": null,+ "sequence": 1,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:01Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-2",+ "role": "tool",+ "content": "",+ "name": "TaskOutput",+ "tool_call_id": "task-1",+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 2,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:02Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-3",+ "role": "assistant",+ "content": "The task finished.",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": [+ {+ "id": "bash-1",+ "type": "function",+ "function": {+ "name": "bash_exec",+ "arguments": "{\"command\": \"ls\"}"+ }+ }+ ],+ "function_call": null,+ "sequence": 3,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:03Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-4",+ "role": "tool",+ "content": "report.md",+ "name": "bash_exec",+ "tool_call_id": "bash-1",+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 4,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:04Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ },+ {+ "id": "row-5",+ "role": "assistant",+ "content": "All done: report.md is ready.",+ "name": null,+ "tool_call_id": null,+ "refusal": null,+ "tool_calls": null,+ "function_call": null,+ "sequence": 5,+ "duration_ms": null,+ "created_at": "2026-09-30T00:00:05Z",+ "model": null,+ "llm_auth_provider": null,+ "llm_credential_id": null,+ "session_id": null,+ "metadata": null+ }+]diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/backend-sim.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/backend-sim.tsnew file mode 100644index 0000000000..a7d8581110--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/backend-sim.ts@@ -0,0 +1,277 @@+import { getGetV2GetSessionMockHandler200 } from "@/app/api/__generated__/endpoints/chat/chat.msw";+import type { SessionDetailResponse } from "@/app/api/__generated__/models/sessionDetailResponse";+import type { SessionDetailResponseMessagesItem } from "@/app/api/__generated__/models/sessionDetailResponseMessagesItem";+import { server } from "@/mocks/mock-server";+import { waitFor } from "@testing-library/react";+import { http, HttpResponse } from "msw";+import fs from "node:fs";+import path from "node:path";+import { expect } from "vitest";+import {+ renderHost,+ TEST_BACKEND_BASE_URL,+ TEST_SESSION_ID,+ typeAndSend,+} from "../sse-helpers";++export type Row = SessionDetailResponseMessagesItem;++export interface RecordedTurn {+ frames: { id: string; sse: string }[];+ rows: Row[];+}++export const STREAM_URL = `${TEST_BACKEND_BASE_URL}/api/chat/sessions/${TEST_SESSION_ID}/stream`;+const FIXTURE_ROOT = path.resolve(+ process.cwd(),+ "../backend/test/fixtures/copilot_stream",+);+const SSE_HEADERS = {+ "content-type": "text/event-stream",+ "x-vercel-ai-ui-message-stream": "v1",+};+const HEARTBEAT_MS = 10_000;+const encoder = new TextEncoder();++/** A turn recorded from the backend pipeline by `stream_drift/recording.py`. */+export function loadRecordedTurn(name: string): RecordedTurn {+ const dir = path.join(FIXTURE_ROOT, name);+ const frames = fs+ .readFileSync(path.join(dir, "frames.jsonl"), "utf8")+ .split("\n")+ .filter(Boolean)+ .map((line) => JSON.parse(line));+ const rows = JSON.parse(fs.readFileSync(path.join(dir, "rows.json"), "utf8"));+ return { frames, rows };+}++interface TurnPlan {+ turn: RecordedTurn;+ /** Rows the session GET returns once this turn has ended; defaults to the+ * recorded ones. A failed persist is a turn whose rows never landed. */+ persistedRows?: Row[];+ /** Start this turn the moment the previous one ends, before its finish is+ * published, as a held call's continuation does. */+ chained?: boolean;+}++/**+ * The backend as the chat sees it: a turn's frames published one by one into+ * a stream every connection replays from its first held entry, and a session+ * GET whose rows and `active_stream` follow the turn.+ */+export function createBackendSim(plans: TurnPlan[]) {+ const state = {+ turnIndex: -1,+ trimmedBefore: 0,+ running: false,+ startedAt: "",+ rows: [] as Row[],+ };+ const published = new Map<number, number>();+ const connections: { closed: boolean; cut: () => void }[] = [];+ const wakers = new Set<() => void>();++ function startTurn(index: number) {+ state.turnIndex = index;+ state.startedAt = new Date().toISOString();+ state.trimmedBefore = 0;+ state.running = true;+ state.rows = [...state.rows, ...leadingUserRows(plans[index].turn.rows)];+ published.set(index, 0);+ }++ // The backend persists, marks the turn completed and wakes any chained+ // turn before it publishes the finish.+ function endTurn(index: number) {+ const plan = plans[index];+ const alreadyPersisted = leadingUserRows(plan.turn.rows).length;+ const turnRows = plan.persistedRows ?? plan.turn.rows;+ state.rows = [...state.rows, ...turnRows.slice(alreadyPersisted)];+ state.running = false;+ if (plans[index + 1]?.chained) startTurn(index + 1);+ }++ /** Publish the running turn's next `count` frames (default: all of them). */+ function publish(count = Infinity) {+ const index = state.turnIndex;+ const frames = plans[index].turn.frames;+ let next = published.get(index) ?? 0;+ const target = Math.min(frames.length, next + count);+ while (next < target) {+ if (isFinish(frames[next])) endTurn(index);+ next += 1;+ published.set(index, next);+ }+ wakers.forEach((wake) => wake());+ }++ function open(turnIndex: number) {+ const frames = plans[turnIndex].turn.frames;+ let index = state.trimmedBefore;+ let cutRequested = false;+ let wake: (() => void) | null = null;+ const connection = {+ closed: false,+ cut() {+ cutRequested = true;+ wake?.();+ },+ };+ connections.push(connection);++ function close(controller: ReadableStreamDefaultController<Uint8Array>) {+ connection.closed = true;+ controller.close();+ }++ const stream = new ReadableStream<Uint8Array>({+ async pull(controller) {+ while (!cutRequested) {+ if (index < (published.get(turnIndex) ?? 0)) {+ const frame = frames[index++];+ controller.enqueue(encoder.encode(frame.sse));+ if (isFinish(frame)) {+ controller.enqueue(encoder.encode("data: [DONE]\n\n"));+ close(controller);+ }+ return;+ }+ const heartbeat = await new Promise<boolean>((resolve) => {+ const timer = setTimeout(() => resolve(true), HEARTBEAT_MS);+ wake = () => {+ clearTimeout(timer);+ resolve(false);+ };+ wakers.add(wake);+ });+ if (wake) wakers.delete(wake);+ wake = null;+ if (heartbeat) {+ controller.enqueue(encoder.encode(": heartbeat\n\n"));+ return;+ }+ }+ close(controller);+ },+ cancel() {+ connection.cut();+ connection.closed = true;+ },+ });+ return new HttpResponse(stream, { status: 200, headers: SSE_HEADERS });+ }++ return {+ handlers: [+ http.post(STREAM_URL, () => {+ startTurn(state.turnIndex + 1);+ return open(state.turnIndex);+ }),+ http.get(STREAM_URL, () => {+ if (!state.running) return new HttpResponse(null, { status: 204 });+ return open(state.turnIndex);+ }),+ ],+ connections,+ /** Mount mid-turn: the turn is already running when the page loads. */+ beginRunning(index = 0) {+ startTurn(index);+ },+ publish,+ /** The running turn's stream lost its first `count` entries. */+ trimHead(count: number) {+ state.trimmedBefore = count;+ },+ cutOpenConnections() {+ connections.filter((c) => !c.closed).forEach((c) => c.cut());+ },+ session(): SessionDetailResponse {+ return {+ id: TEST_SESSION_ID,+ created_at: "2026-09-30T00:00:00Z",+ updated_at: "2026-09-30T00:00:00Z",+ user_id: "test-user",+ has_more_messages: false,+ oldest_sequence: null,+ metadata: { dry_run: false, builder_graph_id: null },+ expert_id: null,+ messages: numbered(state.rows),+ active_stream: state.running+ ? {+ turn_id: `drift-turn-${state.turnIndex}`,+ last_message_id: "0-0",+ started_at: state.startedAt,+ }+ : null,+ chat_status: state.running ? "running" : "idle",+ };+ },+ /** Every row a reload shows once the whole plan has run. */+ finalRows() {+ return numbered(+ plans.flatMap((plan) => plan.persistedRows ?? plan.turn.rows),+ );+ },+ };+}++export type BackendSim = ReturnType<typeof createBackendSim>;++/** Mount the chat against the simulator; the session GET follows its state. */+export function renderAgainst(sim: BackendSim) {+ server.use(...sim.handlers);+ return renderHost({+ sessionResponse: getGetV2GetSessionMockHandler200(() => sim.session()),+ });+}++/** Send the turn's own prompt and wait for its POST to reach the backend. */+export async function sendPrompt(sim: BackendSim, turn: RecordedTurn) {+ const before = sim.connections.length;+ await typeAndSend(String(turn.rows[0].content));+ await waitForConnections(sim, before + 1);+}++export async function waitForConnections(sim: BackendSim, count: number) {+ await waitFor(() => expect(sim.connections.length).toBe(count), {+ timeout: 10_000,+ });+}++/** Past the post-finish probe (500 ms) and the first reconnect delay (1 s),+ * so the client decides while the backend turn is still running. */+export function turnKeepsRunning() {+ return new Promise((resolve) => setTimeout(resolve, 2500));+}++/** The full text of each text block the turn streamed, in order. */+export function streamedTextBlocks(turn: RecordedTurn) {+ const blocks = new Map<string, string>();+ for (const { sse } of turn.frames) {+ if (!sse.startsWith("data: ")) continue;+ const chunk = JSON.parse(sse.slice("data: ".length));+ if (chunk.type === "text-start") blocks.set(chunk.id, "");+ if (chunk.type === "text-delta") {+ blocks.set(chunk.id, (blocks.get(chunk.id) ?? "") + chunk.delta);+ }+ }+ return [...blocks.values()];+}++function isFinish(frame: { sse: string }) {+ return frame.sse.startsWith('data: {"type":"finish"}');+}++function leadingUserRows(rows: Row[]) {+ const firstOther = rows.findIndex((row) => row.role !== "user");+ return firstOther === -1 ? rows : rows.slice(0, firstOther);+}++function numbered(rows: Row[]): Row[] {+ return rows.map((row, sequence) => ({+ ...row,+ id: `row-${sequence}`,+ sequence,+ }));+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/drift-report.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/drift-report.tsnew file mode 100644index 0000000000..3d6a36c22c--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/drift-report.ts@@ -0,0 +1,220 @@+import { server } from "@/mocks/mock-server";+import { cleanup, waitFor } from "@testing-library/react";+import { http, HttpResponse } from "msw";+import { expect, type Mock } from "vitest";+import { resetCopilotChatRegistry } from "../../copilotChatRegistry";+import { useCopilotStreamStore } from "../../copilotStreamStore";+import { renderHost } from "../sse-helpers";+import {+ type BackendSim,+ type RecordedTurn,+ type Row,+ STREAM_URL,+ streamedTextBlocks,+} from "./backend-sim";++interface TranscriptEntry {+ role: "user" | "assistant";+ text: string;+}++/** The invariants every case is held to: `null` when one holds, what the+ * user saw when it does not. */+interface DriftReport {+ /** Every cause here is an expected disconnect, handled silently. */+ connectionToast: string[] | null;+ /** No streamed text block is ever on screen twice. */+ paintedTwice: Record<string, number> | null;+ /** Once settled, the chat shows what a reload from the persisted rows shows. */+ differsFromReload: {+ live: TranscriptEntry[];+ reloaded: TranscriptEntry[];+ } | null;+ /** Text the user watched stream in is still there once the turn ends. */+ lostStreamedText: string[] | null;+ /** Text the stream holds intact is on screen while the turn still runs. */+ missingWhileRunning: string[] | null;+ /** A turn the backend finished cleanly is not shown as failed. */+ errorShown: string | null;+}++type Invariant = keyof DriftReport;++const ERROR_BOX_TEXT = "The assistant encountered an error";++/** Every transcript the page paints, one sample per DOM mutation. */+export function sampleTranscripts() {+ const samples: TranscriptEntry[][] = [];+ const observer = new MutationObserver(() => samples.push(readTranscript()));+ observer.observe(document.body, {+ childList: true,+ subtree: true,+ characterData: true,+ });+ return {+ samples,+ stop: () => observer.disconnect(),+ };+}++/** Which of `texts` the transcript does not show right now. */+export function notOnScreen(texts: string[]) {+ const shown = readTranscript()+ .map((entry) => entry.text)+ .join("\n");+ return texts.filter((text) => !shown.includes(text));+}++/**+ * Settle the live chat, then reload from the persisted rows and compare.+ * Unmounts the live chat, so it is the last step of a case.+ */+export async function reportDrift({+ sim,+ turns,+ painted,+ toast,+ missingWhileRunning = [],+ quietMs = 2500,+}: {+ sim: BackendSim;+ turns: RecordedTurn[];+ painted: ReturnType<typeof sampleTranscripts>;+ toast: Mock;+ /** From `notOnScreen`, taken while the turn ran. */+ missingWhileRunning?: string[];+ quietMs?: number;+}): Promise<DriftReport> {+ const live = await waitForStableTranscript(quietMs, 60_000);+ painted.stop();+ const blocks = turns.flatMap(streamedTextBlocks);+ const shownText = live.map((entry) => entry.text).join("\n");+ const paintedTwice = Object.fromEntries(+ blocks+ .map((block) => [block, maxTimesPainted(painted.samples, block)] as const)+ .filter(([, times]) => times > 1),+ );+ const lost = blocks.filter(+ (block) =>+ maxTimesPainted(painted.samples, block) > 0 && !shownText.includes(block),+ );+ const toasts = connectionToasts(toast);+ const errorShown = document.body.textContent?.includes(ERROR_BOX_TEXT);+ const reloaded = await renderReloaded(sim.finalRows());+ return {+ connectionToast: toasts.length ? toasts : null,+ paintedTwice: Object.keys(paintedTwice).length ? paintedTwice : null,+ differsFromReload:+ JSON.stringify(live) === JSON.stringify(reloaded)+ ? null+ : { live, reloaded },+ lostStreamedText: lost.length ? lost : null,+ missingWhileRunning: missingWhileRunning.length+ ? missingWhileRunning+ : null,+ errorShown: errorShown ? ERROR_BOX_TEXT : null,+ };+}++/**+ * Hold a case to every invariant except the ones `today` names as broken by+ * a live bug. A named invariant that holds again fails the case, so a fix+ * shows up as a red test that asks for its entry to be removed.+ */+export function expectDrift(+ report: DriftReport,+ {+ today = [],+ unchecked = [],+ }: { today?: Invariant[]; unchecked?: Invariant[] },+) {+ const broken = (Object.keys(report) as Invariant[]).filter(+ (name) => report[name] !== null,+ );+ const fixed = today.filter((name) => report[name] === null);+ const unexpected = broken.filter(+ (name) => !today.includes(name) && !unchecked.includes(name),+ );+ expect(+ {+ unexpected: Object.fromEntries(unexpected.map((n) => [n, report[n]])),+ fixed,+ },+ fixed.length+ ? `Now holds: ${fixed.join(", ")}. Remove it from this case's \`today\`.`+ : "An invariant broke that this case does not expect to break.",+ ).toEqual({ unexpected: {}, fixed: [] });+}++export function resetChatState() {+ resetCopilotChatRegistry();+ useCopilotStreamStore.getState().resetAll();+}++/** Wait until the transcript has not changed for `quietMs`. */+export async function waitForStableTranscript(quietMs = 1500, timeout = 15000) {+ let last = JSON.stringify(readTranscript());+ let since = Date.now();+ await waitFor(+ () => {+ const now = JSON.stringify(readTranscript());+ if (now !== last) {+ last = now;+ since = Date.now();+ }+ expect(Date.now() - since).toBeGreaterThanOrEqual(quietMs);+ },+ { timeout, interval: 100 },+ );+ return readTranscript();+}++/** The oracle: what a fresh page load renders from the persisted rows alone. */+async function renderReloaded(rows: Row[]) {+ cleanup();+ resetChatState();+ server.use(+ http.get(STREAM_URL, () => new HttpResponse(null, { status: 204 })),+ );+ renderHost({ sessionOverride: { messages: rows } });+ await waitFor(() => expect(readTranscript().length).toBeGreaterThan(0), {+ timeout: 10000,+ });+ return waitForStableTranscript(500);+}++/**+ * One entry per rendered message, in order. The turn's "Thought for" label is+ * left out: the client counts it live and the server stamps its own figure.+ */+function readTranscript(): TranscriptEntry[] {+ return Array.from(document.querySelectorAll("[data-message-id]")).map(+ (el) => ({+ role: el.classList.contains("is-user") ? "user" : "assistant",+ text: (el.textContent ?? "")+ .replace(/Thought for (\d+m )?\d+s/g, "")+ .replace(/\s+/g, " ")+ .trim(),+ }),+ );+}++function maxTimesPainted(samples: TranscriptEntry[][], text: string) {+ return Math.max(+ 0,+ ...samples.map(+ (sample) =>+ sample+ .map((entry) => entry.text)+ .join("\n")+ .split(text).length - 1,+ ),+ );+}++function connectionToasts(toast: Mock) {+ return toast.mock.calls+ .map(([options]) => options ?? {})+ .filter(({ title }) => String(title).startsWith("Connection"))+ .map(({ title, description }) => `${title}: ${description}`);+}diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-content.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-content.test.tsxnew file mode 100644index 0000000000..5cfae1677b--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-content.test.tsx@@ -0,0 +1,160 @@+import { afterEach, beforeEach, describe, it, vi } from "vitest";+import { TEST_BACKEND_BASE_URL } from "../sse-helpers";+import {+ createBackendSim,+ loadRecordedTurn,+ renderAgainst,+ sendPrompt,+ turnKeepsRunning,+ waitForConnections,+} from "./backend-sim";+import {+ expectDrift,+ notOnScreen,+ reportDrift,+ resetChatState,+ sampleTranscripts,+} from "./drift-report";++vi.mock("@/services/environment", async (importActual) => {+ const actual = await importActual<typeof import("@/services/environment")>();+ return {+ ...actual,+ environment: {+ ...actual.environment,+ getAGPTServerBaseUrl: () => TEST_BACKEND_BASE_URL,+ },+ };+});+vi.mock("../../helpers", async (importActual) => {+ const actual = await importActual<typeof import("../../helpers")>();+ return { ...actual, getCopilotAuthHeaders: async () => ({}) };+});+vi.mock("@/lib/auth/hooks/useAuth", () => ({+ useAuth: () => ({ isUserLoading: false, isLoggedIn: true }),+}));+vi.mock("@/services/feature-flags/use-get-flag", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/services/feature-flags/use-get-flag")+ >();+ return { ...actual, useGetFlag: () => false };+});+const { toastMock } = vi.hoisted(() => ({ toastMock: vi.fn() }));+vi.mock("@/components/molecules/Toast/use-toast", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/components/molecules/Toast/use-toast")+ >();+ return { ...actual, toast: toastMock };+});++const toolTurn = loadRecordedTurn("baseline-tool-turn");+const textTurn = loadRecordedTurn("dummy-text-turn");+// Frame 6 is the first block's second delta, 12 the tool's output, and+// 13-17 the whole second text block.+const SECOND_DELTA = 6;+const TOOL_OUTPUT_DONE = 13;+const SECOND_BLOCK_DONE = 18;++beforeEach(() => {+ resetChatState();+ toastMock.mockClear();+});++afterEach(() => {+ vi.useRealTimers();+ resetChatState();+});++describe("stream drift — what the stream and the rows hold", () => {+ it(+ "W5: a reload into a turn whose stream lost its head still renders the turn",+ { timeout: 90_000 },+ async () => {+ const sim = createBackendSim([{ turn: toolTurn }]);+ sim.beginRunning();+ sim.publish(TOOL_OUTPUT_DONE);+ // Simulated: the length cap dropped the stream's first entries, so the+ // replay opens on a text-delta whose text-start is gone.+ sim.trimHead(SECOND_DELTA);+ renderAgainst(sim);+ const painted = sampleTranscripts();+ await waitForConnections(sim, 1);+ // The second block reaches the stream whole, start included.+ sim.publish(SECOND_BLOCK_DONE - TOOL_OUTPUT_DONE);+ await turnKeepsRunning();+ const missingWhileRunning = notOnScreen(["It says Example Domain."]);+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [toolTurn],+ painted,+ toast: toastMock,+ missingWhileRunning,+ quietMs: 8000,+ }),+ // The parser throws on the orphan delta and every reconnect replays+ // the same head, so nothing of the running turn renders and the+ // reconnects announce "Connection lost". Only the turn's end repairs+ // the transcript, from the rows.+ { today: ["connectionToast", "missingWhileRunning"] },+ );+ },+ );++ it(+ "approval wake: a reload into a turn no executor runs raises no connection alarm",+ { timeout: 60_000 },+ async () => {+ const [prompt] = textTurn.rows;+ // The executor dropped the turn, so its stream never gets an entry.+ const sim = createBackendSim([{ turn: { frames: [], rows: [prompt] } }]);+ sim.beginRunning();+ vi.useFakeTimers({ shouldAdvanceTime: true });+ renderAgainst(sim);+ const painted = sampleTranscripts();+ await waitForConnections(sim, 1);+ // Heartbeats keep arriving every 10 s: the connection is alive, only+ // the turn is dead, and only the server can say so.+ await vi.advanceTimersByTimeAsync(60_000);++ expectDrift(+ await reportDrift({ sim, turns: [], painted, toast: toastMock }),+ // The 6 s restore watchdog sees no content, reconnects three times and+ // ends on "Unable to reconnect. Please refresh the page."+ { today: ["connectionToast"] },+ );+ },+ );++ it(+ "W8: a reply whose final persist failed stays on screen",+ { timeout: 60_000 },+ async () => {+ const [prompt] = textTurn.rows;+ const sim = createBackendSim([+ { turn: textTurn, persistedRows: [prompt] },+ ]);+ renderAgainst(sim);+ await sendPrompt(sim, textTurn);+ const painted = sampleTranscripts();+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [textTurn],+ painted,+ toast: toastMock,+ }),+ // The end-of-turn hydrate replaces the list with rows that lack the+ // reply, so it vanishes in front of the user. The rows are the fault+ // here, so a reload is no oracle for this case.+ { today: ["lostStreamedText"], unchecked: ["differsFromReload"] },+ );+ },+ );+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-guards.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-guards.test.tsxnew file mode 100644index 0000000000..61ad8c0559--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-guards.test.tsx@@ -0,0 +1,124 @@+import { afterEach, beforeEach, describe, it, vi } from "vitest";+import { TEST_BACKEND_BASE_URL } from "../sse-helpers";+import {+ createBackendSim,+ loadRecordedTurn,+ renderAgainst,+ sendPrompt,+ turnKeepsRunning,+ streamedTextBlocks,+ waitForConnections,+} from "./backend-sim";+import {+ expectDrift,+ notOnScreen,+ reportDrift,+ resetChatState,+ sampleTranscripts,+ waitForStableTranscript,+} from "./drift-report";++vi.mock("@/services/environment", async (importActual) => {+ const actual = await importActual<typeof import("@/services/environment")>();+ return {+ ...actual,+ environment: {+ ...actual.environment,+ getAGPTServerBaseUrl: () => TEST_BACKEND_BASE_URL,+ },+ };+});+vi.mock("../../helpers", async (importActual) => {+ const actual = await importActual<typeof import("../../helpers")>();+ return { ...actual, getCopilotAuthHeaders: async () => ({}) };+});+vi.mock("@/lib/auth/hooks/useAuth", () => ({+ useAuth: () => ({ isUserLoading: false, isLoggedIn: true }),+}));+vi.mock("@/services/feature-flags/use-get-flag", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/services/feature-flags/use-get-flag")+ >();+ return { ...actual, useGetFlag: () => false };+});+const { toastMock } = vi.hoisted(() => ({ toastMock: vi.fn() }));+vi.mock("@/components/molecules/Toast/use-toast", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/components/molecules/Toast/use-toast")+ >();+ return { ...actual, toast: toastMock };+});++beforeEach(() => {+ resetChatState();+ toastMock.mockClear();+});++afterEach(() => resetChatState());++describe("stream drift — guards", () => {+ it.each([+ "dummy-text-turn",+ "baseline-tool-turn",+ // A tool result that lands after the next text: the backend ends the+ // text block before the step closes (#14762), or the parser dies.+ "sdk-late-tool-result",+ ])(+ "a sent turn renders what a reload renders (%s)",+ { timeout: 60_000 },+ async (name) => {+ const turn = loadRecordedTurn(name);+ const sim = createBackendSim([{ turn }]);+ renderAgainst(sim);+ await sendPrompt(sim, turn);+ const painted = sampleTranscripts();+ sim.publish(turn.frames.length - 1);+ await waitForStableTranscript(1000);+ const missingWhileRunning = notOnScreen(streamedTextBlocks(turn));+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [turn],+ painted,+ toast: toastMock,+ missingWhileRunning,+ }),+ {},+ );+ },+ );++ it(+ "a reload into a running turn resumes it once",+ { timeout: 60_000 },+ async () => {+ const turn = loadRecordedTurn("baseline-tool-turn");+ const sim = createBackendSim([{ turn }]);+ sim.beginRunning();+ sim.publish(9);+ renderAgainst(sim);+ const painted = sampleTranscripts();+ await waitForConnections(sim, 1);+ await turnKeepsRunning();+ sim.publish(turn.frames.length - 1 - 9);+ await waitForStableTranscript(1000);+ const missingWhileRunning = notOnScreen(streamedTextBlocks(turn));+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [turn],+ painted,+ toast: toastMock,+ missingWhileRunning,+ }),+ {},+ );+ },+ );+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-reconnects.test.tsx b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-reconnects.test.tsxnew file mode 100644index 0000000000..e359e2f75d--- /dev/null+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/drift/stream-drift-reconnects.test.tsx@@ -0,0 +1,217 @@+import { screen } from "@testing-library/react";+import { afterEach, beforeEach, describe, it, vi } from "vitest";+import { TEST_BACKEND_BASE_URL } from "../sse-helpers";+import {+ createBackendSim,+ loadRecordedTurn,+ renderAgainst,+ sendPrompt,+ turnKeepsRunning,+ waitForConnections,+} from "./backend-sim";+import {+ expectDrift,+ reportDrift,+ resetChatState,+ sampleTranscripts,+} from "./drift-report";++vi.mock("@/services/environment", async (importActual) => {+ const actual = await importActual<typeof import("@/services/environment")>();+ return {+ ...actual,+ environment: {+ ...actual.environment,+ getAGPTServerBaseUrl: () => TEST_BACKEND_BASE_URL,+ },+ };+});+vi.mock("../../helpers", async (importActual) => {+ const actual = await importActual<typeof import("../../helpers")>();+ return { ...actual, getCopilotAuthHeaders: async () => ({}) };+});+vi.mock("@/lib/auth/hooks/useAuth", () => ({+ useAuth: () => ({ isUserLoading: false, isLoggedIn: true }),+}));+vi.mock("@/services/feature-flags/use-get-flag", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/services/feature-flags/use-get-flag")+ >();+ return { ...actual, useGetFlag: () => false };+});+const { toastMock } = vi.hoisted(() => ({ toastMock: vi.fn() }));+vi.mock("@/components/molecules/Toast/use-toast", async (importActual) => {+ const actual =+ await importActual<+ typeof import("@/components/molecules/Toast/use-toast")+ >();+ return { ...actual, toast: toastMock };+});++const toolTurn = loadRecordedTurn("baseline-tool-turn");+const textTurn = loadRecordedTurn("dummy-text-turn");+// Frames 0-8 end the first text block; 10-11 open the tool call.+const FIRST_TEXT_BLOCK_DONE = 9;+const TOOL_CALL_OPEN = 12;++beforeEach(() => {+ resetChatState();+ toastMock.mockClear();+});++afterEach(() => {+ vi.useRealTimers();+ setTabVisibility("visible");+ resetChatState();+});++describe("stream drift — a second connection onto a live turn", () => {+ it(+ "W1: a tab shown again after 30 s resumes over the stream it never lost",+ { timeout: 60_000 },+ async () => {+ const sim = createBackendSim([{ turn: toolTurn }]);+ renderAgainst(sim);+ await sendPrompt(sim, toolTurn);+ const painted = sampleTranscripts();+ sim.publish(FIRST_TEXT_BLOCK_DONE);+ await screen.findByText("Let me fetch that page.", undefined, {+ timeout: 5000,+ });++ // Desktop browsers keep a hidden tab's fetch open, so the POST stream is+ // still healthy when the wake re-sync fires.+ vi.useFakeTimers({ toFake: ["Date"], shouldAdvanceTime: true });+ setTabVisibility("hidden");+ vi.setSystemTime(Date.now() + 31_000);+ setTabVisibility("visible");+ vi.useRealTimers();+ await waitForConnections(sim, 2);+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [toolTurn],+ painted,+ toast: toastMock,+ }),+ // The resume replays the turn into the list while the POST stream+ // still writes it: both copies of the first block are on screen.+ { today: ["paintedTwice"] },+ );+ },+ );++ it(+ "W7: a tool call silent for 70 s keeps its stream, heartbeats included",+ { timeout: 60_000 },+ async () => {+ const sim = createBackendSim([{ turn: toolTurn }]);+ renderAgainst(sim);+ await sendPrompt(sim, toolTurn);+ const painted = sampleTranscripts();+ sim.publish(FIRST_TEXT_BLOCK_DONE);+ await screen.findByText("Let me fetch that page.", undefined, {+ timeout: 5000,+ });++ vi.useFakeTimers({ shouldAdvanceTime: true });+ sim.publish(TOOL_CALL_OPEN - FIRST_TEXT_BLOCK_DONE);+ // The route writes a heartbeat comment every 10 s; the parser drops them.+ await vi.advanceTimersByTimeAsync(70_000);+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [toolTurn],+ painted,+ toast: toastMock,+ }),+ // The 60 s stall watchdog reads the silence as a dead stream and+ // reconnects with a toast.+ { today: ["connectionToast"] },+ );+ },+ );+});++describe("stream drift — a stream that ends while its turn runs on", () => {+ it(+ "W4: the load balancer's 30-minute cut is resumed silently",+ { timeout: 60_000 },+ async () => {+ const sim = createBackendSim([{ turn: toolTurn }]);+ renderAgainst(sim);+ await sendPrompt(sim, toolTurn);+ const painted = sampleTranscripts();+ sim.publish(FIRST_TEXT_BLOCK_DONE);+ await screen.findByText("Let me fetch that page.", undefined, {+ timeout: 5000,+ });++ // Simulated: the cut is a clean close with no finish, whatever the age.+ sim.cutOpenConnections();+ await waitForConnections(sim, 2);+ await turnKeepsRunning();+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [toolTurn],+ painted,+ toast: toastMock,+ }),+ // Two resumes attach to one running turn: the once-per-mount resume,+ // and the post-finish probe's reconnect with "Connection lost".+ // Neither aborts the other, so the first block is painted three times.+ { today: ["connectionToast", "paintedTwice"] },+ );+ },+ );++ it(+ "chained turn: the turn a finished turn wakes is attached to quietly",+ { timeout: 60_000 },+ async () => {+ const continuation = {+ ...textTurn,+ rows: textTurn.rows.filter((row) => row.role !== "user"),+ };+ const sim = createBackendSim([+ { turn: toolTurn },+ { turn: continuation, chained: true },+ ]);+ renderAgainst(sim);+ await sendPrompt(sim, toolTurn);+ const painted = sampleTranscripts();+ // The first turn's end starts the continuation before its finish lands.+ sim.publish();+ await waitForConnections(sim, 2);+ await turnKeepsRunning();+ sim.publish();++ expectDrift(+ await reportDrift({+ sim,+ turns: [toolTurn, continuation],+ painted,+ toast: toastMock,+ }),+ // The probe cannot tell the new turn from a dropped stream (#15044).+ { today: ["connectionToast"] },+ );+ },+ );+});++function setTabVisibility(state: "hidden" | "visible") {+ Object.defineProperty(document, "visibilityState", {+ value: state,+ configurable: true,+ });+ document.dispatchEvent(new Event("visibilitychange"));+}