AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(copilot): an approval's continuation reply streams into the chat that approved it (#14974)
Reinier van der Leeraa3d44bc72f1
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 3ba0da4d3f..72cac07ba1 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -1617,6 +1617,10 @@ async def cancel_session_task(await asyncio.sleep(_CANCEL_CONFIRM_POLL_INTERVAL_SECONDS)waited += _CANCEL_CONFIRM_POLL_INTERVAL_SECONDSsession_state = await stream_registry.get_session(session_id)+ # A turn the cancelled one's end woke is not the one the user stopped,+ # and its queued follow-ups are its own.+ if session_state and session_state.turn_id != active_session.turn_id:+ return CancelSessionResponse(cancelled=True)if session_state is None or session_state.status != "running":logger.info(f"[CANCEL] Session ...{session_id[-8:]} confirmed stopped "@@ -1638,6 +1642,7 @@ async def cancel_session_task(session_id,error_message=stream_registry.CANCELLED_MESSAGE,skip_error_publish=True,+ turn_id=active_session.turn_id,)# Status is now force-flipped out of "running"; re-clear to drop any# follow-up that landed during the poll window.diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.pyindex 44e22c0090..4c3bc210c4 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py@@ -3236,9 +3236,56 @@ def test_cancel_session_timeout_completes_as_cancelled_not_as_an_error(mock_registry.mark_session_completed.await_args.kwargs.get("skip_error_publish")is True), "a user cancel must not be published to the stream as an error"+ assert mock_registry.mark_session_completed.await_args.kwargs["turn_id"] == "turn-1"assert response.json()["reason"] == "cancel_published_not_confirmed"+def test_cancel_confirms_once_the_next_turn_holds_the_session(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """The cancelled turn's end can wake the next turn; the poll must not+ force-complete that one as the user's cancel."""+ from backend.copilot.stream_registry import ActiveSession++ _mock_validate_session(mocker)+ mocker.patch(+ "backend.copilot.turn_queue.cancel_queued_turn",+ new=AsyncMock(return_value=False),+ )++ def turn(turn_id: str) -> ActiveSession:+ return ActiveSession(+ session_id="sess-1",+ user_id=TEST_USER_ID,+ tool_call_id="chat_stream",+ tool_name="chat",+ turn_id=turn_id,+ status="running",+ )++ mock_registry = MagicMock()+ mock_registry.get_active_session = AsyncMock(return_value=(turn("turn-1"), "1-0"))+ mock_registry.get_session = AsyncMock(return_value=turn("turn-2"))+ mock_registry.mark_session_completed = AsyncMock(return_value=True)+ mocker.patch("backend.api.features.chat.routes.stream_registry", mock_registry)+ mocker.patch(+ "backend.api.features.chat.routes.enqueue_cancel_task",+ new_callable=AsyncMock,+ )+ mocker.patch.object(chat_routes, "_CANCEL_CONFIRM_TIMEOUT_SECONDS", 0.02)+ mocker.patch.object(chat_routes, "_CANCEL_CONFIRM_POLL_INTERVAL_SECONDS", 0.01)+ clear_pending = mocker.patch.object(+ chat_routes, "_clear_pending_best_effort", new_callable=AsyncMock+ )++ response = client.post("/sessions/sess-1/cancel")++ assert response.json()["cancelled"] is True+ mock_registry.mark_session_completed.assert_not_awaited()+ # Only the up-front clear; the next turn's follow-ups stay queued.+ clear_pending.assert_awaited_once()++def test_cancel_session_clears_pending_buffer(mocker: pytest_mock.MockerFixture,) -> None:diff --git a/autogpt_platform/backend/backend/copilot/executor/processor.py b/autogpt_platform/backend/backend/copilot/executor/processor.pyindex cbfe6c942b..054619e3cc 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor.py@@ -94,6 +94,7 @@ EXPERT_SETUP_POLL_SECONDS = 1.0def sync_fail_close_session(session_id: str,+ turn_id: str,log: "CoPilotLogMetadata | TruncatedLogger",execution_loop: asyncio.AbstractEventLoop,) -> None:@@ -122,7 +123,7 @@ def sync_fail_close_session(async def _bounded() -> None:await asyncio.wait_for(stream_registry.mark_session_completed(- session_id, error_message=SHUTDOWN_ERROR_MESSAGE+ session_id, error_message=SHUTDOWN_ERROR_MESSAGE, turn_id=turn_id),timeout=_FAIL_CLOSE_REDIS_TIMEOUT,)@@ -422,7 +423,9 @@ class CoPilotProcessor:try:self._execute(entry, cancel, cluster_lock, log)finally:- sync_fail_close_session(entry.session_id, log, self.execution_loop)+ sync_fail_close_session(+ entry.session_id, entry.turn_id, log, self.execution_loop+ )elapsed = time.monotonic() - start_timelog.info(f"Execution completed in {elapsed:.2f}s")@@ -805,7 +808,7 @@ class CoPilotProcessor:finally:try:await stream_registry.mark_session_completed(- entry.session_id, error_message=error_msg+ entry.session_id, error_message=error_msg, turn_id=entry.turn_id)except Exception as mark_err:log.error(f"Failed to mark session completed: {mark_err}")diff --git a/autogpt_platform/backend/backend/copilot/executor/processor_test.py b/autogpt_platform/backend/backend/copilot/executor/processor_test.pyindex b0ca9f4ac3..b4f63f7270 100644--- a/autogpt_platform/backend/backend/copilot/executor/processor_test.py+++ b/autogpt_platform/backend/backend/copilot/executor/processor_test.py@@ -651,7 +651,7 @@ async def test_unowned_expert_session_fails_before_engine_work() -> None:sdk_engine.assert_not_called()upsert.assert_not_awaited()mark_completed.assert_awaited_once_with(- "sess-1", error_message="expert is not owned by user"+ "sess-1", error_message="expert is not owned by user", turn_id="turn-1")@@ -724,7 +724,9 @@ async def test_expert_tenancy_errors_publish_actionable_copy(dummy_engine.assert_not_called()baseline_engine.assert_not_called()sdk_engine.assert_not_called()- mark_completed.assert_awaited_once_with("sess-1", error_message=expected_message)+ mark_completed.assert_awaited_once_with(+ "sess-1", error_message=expected_message, turn_id="turn-1"+ )def _codex_entry(@@ -961,6 +963,7 @@ async def test_codex_release_failure_does_not_fail_successful_turn():mark_completed.assert_awaited_once_with("sess-codex",error_message=None,+ turn_id="turn-codex",)@@ -1014,6 +1017,7 @@ async def test_codex_checkpoint_failure_fails_closed_after_successful_turn():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_credential_checkpoint_failed",+ turn_id="turn-codex",)@@ -1048,6 +1052,7 @@ async def test_codex_queue_route_mismatch_fails_before_credential_acquire():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_session_route_mismatch",+ turn_id="turn-codex",)@@ -1089,6 +1094,7 @@ async def test_codex_entitlement_is_checked_before_credential_acquire():mark_completed.assert_awaited_once_with("sess-codex",error_message="Max plan required",+ turn_id="turn-codex",)@@ -1136,6 +1142,7 @@ async def test_codex_busy_credential_fails_closed_without_platform_fallback():mark_completed.assert_awaited_once_with("sess-codex",error_message="codex_credential_busy",+ turn_id="turn-codex",)@@ -1171,12 +1178,13 @@ class TestSyncFailCloseSession:"backend.copilot.executor.processor.stream_registry.mark_session_completed",new=mock_mark,):- sync_fail_close_session("sess-1", _make_log(), exec_loop)+ sync_fail_close_session("sess-1", "turn-1", _make_log(), exec_loop)mock_mark.assert_awaited_once()assert mock_mark.await_args is not Noneassert mock_mark.await_args.args[0] == "sess-1"assert "shut down" in mock_mark.await_args.kwargs["error_message"].lower()+ assert mock_mark.await_args.kwargs["turn_id"] == "turn-1"def test_swallows_redis_error(self, exec_loop) -> None:# Raising from the mock ensures the helper catches the exception@@ -1186,7 +1194,9 @@ class TestSyncFailCloseSession:"backend.copilot.executor.processor.stream_registry.mark_session_completed",new=mock_mark,):- sync_fail_close_session("sess-2", _make_log(), exec_loop) # must not raise+ sync_fail_close_session(+ "sess-2", "turn-2", _make_log(), exec_loop+ ) # must not raisemock_mark.assert_awaited_once()@@ -1203,7 +1213,9 @@ class TestSyncFailCloseSession:new=mock_mark,):# Must not raise even though the loop is closed- sync_fail_close_session("sess-closed-loop", _make_log(), dead_loop)+ sync_fail_close_session(+ "sess-closed-loop", "turn-3", _make_log(), dead_loop+ )# mark_session_completed was never scheduled because the loop was deadmock_mark.assert_not_awaited()@@ -1229,7 +1241,7 @@ class TestSyncFailCloseSession:):start = _time.monotonic()sync_fail_close_session(- "sess-hang", _make_log(), exec_loop+ "sess-hang", "turn-4", _make_log(), exec_loop) # must not raiseelapsed = _time.monotonic() - startdiff --git a/autogpt_platform/backend/backend/copilot/stream_registry.py b/autogpt_platform/backend/backend/copilot/stream_registry.pyindex 71442f2248..0221447f42 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry.py@@ -888,6 +888,7 @@ async def mark_session_completed(error_message: str | None = None,*,skip_error_publish: bool = False,+ turn_id: str = "",) -> bool:"""Mark a session as completed, then publish StreamFinish.@@ -908,6 +909,8 @@ async def mark_session_completed(cancel, which the frontend would otherwise render as "the assistantencountered an error", and when the error has already beenpublished to the stream (e.g. via stream_and_publish).+ turn_id: The finishing turn. When given, a session whose meta already+ belongs to a later turn is left alone.Returns:True if session was newly marked completed, False if already completed/failed@@ -918,11 +921,14 @@ async def mark_session_completed(# Resolve turn_id for publishing to the correct streammeta: dict[Any, Any] = await redis.hgetall(meta_key) # type: ignore[misc]- turn_id = _parse_session_meta(meta, session_id).turn_id if meta else session_id+ guard = ("turn_id", turn_id) if turn_id else None+ if not turn_id:+ turn_id = _parse_session_meta(meta, session_id).turn_id if meta else session_id- # Atomic compare-and-swap: only update if status is "running"+ # Atomic compare-and-swap: only update if status is "running". A turn's end+ # can wake the next one, so its late safety-net call must not close that.swapped = await hash_compare_and_set(- redis, meta_key, "status", expected="running", new=status+ redis, meta_key, "status", expected="running", new=status, guard=guard)# Clean up the in-memory TTL refresh tracker to prevent unbounded growth.@@ -1164,6 +1170,7 @@ async def get_active_session(await mark_session_completed(session_id,error_message=f"Session timed out after {age_seconds:.0f}s",+ turn_id=_parse_session_meta(meta, session_id).turn_id,)return None, "0-0"except (ValueError, TypeError) as e:diff --git a/autogpt_platform/backend/backend/copilot/stream_registry_test.py b/autogpt_platform/backend/backend/copilot/stream_registry_test.pyindex 50d2abd033..4265563c45 100644--- a/autogpt_platform/backend/backend/copilot/stream_registry_test.py+++ b/autogpt_platform/backend/backend/copilot/stream_registry_test.py@@ -2,6 +2,7 @@import asyncioimport json+import uuidfrom unittest.mock import AsyncMock, patchimport pytest@@ -10,6 +11,8 @@ from redis.exceptions import RedisErrorfrom backend.copilot import stream_registryfrom backend.copilot.constants import STREAM_LOCK_PREFIXfrom backend.copilot.executor.utils import get_session_lock_key+from backend.data import redis_client+from backend.util.testing import is_tcp_port_reachabledef test_tool_display_roundtrips_redis_and_sse():@@ -840,3 +843,72 @@ async def test_a_finished_turn_wakes_the_chat_for_cards_answered_during_it(assert wake.await_count == (1 if wakes else 0)if wakes:wake.assert_awaited_once_with("u1", "sess-1")+++ not is_tcp_port_reachable(redis_client.HOST, redis_client.PORT),+ reason="no local Redis reachable; the CAS script needs one to run",+)+class TestCompletionOnRealRedis:+ """A turn's end can wake the next turn before its safety net runs."""++ @pytest.fixture+ async def session_id(self):+ session_id = f"test-completion-{uuid.uuid4().hex}"+ yield session_id+ redis = await redis_client.get_redis_async()+ await redis.delete(stream_registry.get_session_meta_key(session_id))++ async def test_a_late_completion_leaves_the_next_turn_running(self, session_id):+ await stream_registry.create_session(session_id, None, "", "", turn_id="a")+ await stream_registry.create_session(session_id, None, "", "", turn_id="b")++ with patch.object(stream_registry, "publish_chunk", new=AsyncMock()) as publish:+ closed = await stream_registry.mark_session_completed(+ session_id, error_message="shut down", turn_id="a"+ )++ session = await stream_registry.get_session(session_id)+ assert closed is False+ assert session is not None and session.status == "running"+ publish.assert_not_awaited()++ async def test_a_turn_still_completes_itself(self, session_id):+ await stream_registry.create_session(session_id, None, "", "", turn_id="a")++ with (+ patch.object(stream_registry, "publish_chunk", new=AsyncMock()),+ patch.object(+ stream_registry.chat_db(),+ "set_turn_duration",+ new=AsyncMock(),+ create=True,+ ),+ ):+ closed = await stream_registry.mark_session_completed(+ session_id, turn_id="a"+ )++ session = await stream_registry.get_session(session_id)+ assert closed is True+ assert session is not None and session.status == "completed"++ async def test_meta_without_a_turn_id_still_completes(self, session_id):+ await stream_registry.create_session(session_id, None, "", "", turn_id="a")+ redis = await redis_client.get_redis_async()+ await redis.hdel(stream_registry.get_session_meta_key(session_id), "turn_id")++ with (+ patch.object(stream_registry, "publish_chunk", new=AsyncMock()),+ patch.object(+ stream_registry.chat_db(),+ "set_turn_duration",+ new=AsyncMock(),+ create=True,+ ),+ ):+ closed = await stream_registry.mark_session_completed(+ session_id, turn_id="a"+ )++ assert closed is Truediff --git a/autogpt_platform/backend/backend/data/redis_helpers.py b/autogpt_platform/backend/backend/data/redis_helpers.pyindex bbdaa0a220..71059eb275 100644--- a/autogpt_platform/backend/backend/data/redis_helpers.py+++ b/autogpt_platform/backend/backend/data/redis_helpers.py@@ -38,13 +38,19 @@ from backend.data.redis_client import AsyncRedisClient, RedisClient# ARGV[1] hash field# ARGV[2] expected current value# ARGV[3] new value+# ARGV[4] optional guard field which, when set and non-empty, must equal ARGV[5]_HASH_CAS_LUA = """-local current = redis.call('HGET', KEYS[1], ARGV[1])-if current == ARGV[2] then- redis.call('HSET', KEYS[1], ARGV[1], ARGV[3])- return 1+if redis.call('HGET', KEYS[1], ARGV[1]) ~= ARGV[2] then+ return 0end-return 0+if #ARGV > 3 then+ local guard = redis.call('HGET', KEYS[1], ARGV[4])+ if guard and guard ~= '' and guard ~= ARGV[5] then+ return 0+ end+end+redis.call('HSET', KEYS[1], ARGV[1], ARGV[3])+return 1"""# Push to a capped list only when a hash field currently matches the expected@@ -409,8 +415,10 @@ async def hash_compare_and_set(*,expected: str,new: str,+ guard: tuple[str, str] | None = None,) -> bool:- """Atomically set ``HSET key field new`` iff current value == *expected*.+ """Atomically set ``HSET key field new`` iff current value == *expected*+ (and, with ``guard=(field, value)``, that field is unset, empty or value).Returns ``True`` if the swap happened, ``False`` otherwise.@@ -422,6 +430,6 @@ async def hash_compare_and_set("""result = await cast("Any",- redis.eval(_HASH_CAS_LUA, 1, key, field, expected, new),+ redis.eval(_HASH_CAS_LUA, 1, key, field, expected, new, *(guard or ())),)return int(result) == 1diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useFollowBackendTurn.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useFollowBackendTurn.test.tsindex 5504270df8..e50a5a7f39 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useFollowBackendTurn.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/__tests__/useFollowBackendTurn.test.ts@@ -1,10 +1,19 @@import { act, renderHook } from "@testing-library/react";-import { expect, test, vi } from "vitest";+import { afterEach, expect, test, vi } from "vitest";import { useFollowBackendTurn } from "../useFollowBackendTurn";-function setup(status: string) {+const idle = { data: { status: 200, data: { active_stream: null } } };+const running = {+ data: { status: 200, data: { active_stream: { turn_id: "t" } } },+};++afterEach(() => {+ vi.useRealTimers();+});++function setup(status: string, results: { data?: unknown }[] = [running]) {const hasResumedRef = { current: true };- const refetchSession = vi.fn(async () => ({}));+ const refetchSession = vi.fn(async () => results.shift() ?? idle);const hook = renderHook(({ status }) =>useFollowBackendTurn({ status, refetchSession, hasResumedRef }),@@ -13,24 +22,66 @@ function setup(status: string) {return { hook, hasResumedRef, refetchSession };}-test("an idle chat re-arms the resume and refetches at once", () => {+test("an idle chat re-arms the resume and refetches at once", async () => {const { hook, hasResumedRef, refetchSession } = setup("ready");- act(() => hook.result.current.followBackendTurn());+ await act(async () => hook.result.current.followBackendTurn());expect(hasResumedRef.current).toBe(false);expect(refetchSession).toHaveBeenCalledTimes(1);});-test("a running turn is followed only once it ends", () => {+test("a running turn is followed only once it ends", async () => {const { hook, hasResumedRef, refetchSession } = setup("streaming");- act(() => hook.result.current.followBackendTurn());+ await act(async () => hook.result.current.followBackendTurn());expect(refetchSession).not.toHaveBeenCalled();expect(hasResumedRef.current).toBe(true);- hook.rerender({ status: "ready" });+ await act(async () => hook.rerender({ status: "ready" }));expect(refetchSession).toHaveBeenCalledTimes(1);expect(hasResumedRef.current).toBe(false);});++test("a turn that ended while the answer was posting is still followed", async () => {+ const { hook, refetchSession } = setup("streaming");+ const clickedWhileStreaming = hook.result.current.followBackendTurn;++ await act(async () => hook.rerender({ status: "ready" }));+ await act(async () => clickedWhileStreaming());++ expect(refetchSession).toHaveBeenCalledTimes(1);+});++test("the probe retries until the answer's turn has a stream", async () => {+ vi.useFakeTimers();+ const { hook, refetchSession } = setup("ready", [idle, idle, running]);++ await act(async () => hook.result.current.followBackendTurn());+ await act(async () => vi.advanceTimersByTimeAsync(5_000));++ expect(refetchSession).toHaveBeenCalledTimes(3);+});++test("a second answer mid-probe does not start a second probe", async () => {+ vi.useFakeTimers();+ let inFlight = 0;+ let maxInFlight = 0;+ const hasResumedRef = { current: true };+ async function refetchSession() {+ maxInFlight = Math.max(maxInFlight, ++inFlight);+ await new Promise((r) => setTimeout(r, 200));+ inFlight--;+ return idle;+ }+ const hook = renderHook(() =>+ useFollowBackendTurn({ status: "ready", refetchSession, hasResumedRef }),+ );++ await act(async () => hook.result.current.followBackendTurn());+ await act(async () => hook.result.current.followBackendTurn());+ await act(async () => vi.advanceTimersByTimeAsync(10_000));++ expect(maxInFlight).toBe(1);+});diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/useFollowBackendTurn.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/useFollowBackendTurn.tsindex 918df3956f..775887e79a 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/useFollowBackendTurn.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/useFollowBackendTurn.ts@@ -1,4 +1,5 @@import { useEffect, useRef } from "react";+import { hasActiveBackendStream } from "./helpers";interface Args {status: string;@@ -6,6 +7,11 @@ interface Args {hasResumedRef: React.MutableRefObject<boolean>;}+// The answer's turn is dispatched as the POST returns, or when a running turn+// ends; its stream can register a beat after either, so the probe retries.+const FOLLOW_ATTEMPTS = 8;+const FOLLOW_INTERVAL_MS = 500;+// An answered approval card starts the chat's next turn on the server. The page// resumes a server-started stream once per mount, after hydration; re-arming// that resume lets the refetched rows land first, so the replay follows them.@@ -16,22 +22,60 @@ export function useFollowBackendTurn({}: Args) {const pendingRef = useRef(false);const isBusy = status === "streaming" || status === "submitted";+ // Read live: the caller holds this hook's function from the render where+ // the card was clicked, and the turn may have ended during the POST.+ const isBusyRef = useRef(isBusy);+ isBusyRef.current = isBusy;+ const isMountedRef = useRef(true);+ // One probe at a time; a second answer mid-probe restarts its budget.+ const attemptsLeftRef = useRef(0);+ const isProbingRef = useRef(false);+ const timerRef = useRef<{ id: number; wake: () => void } | null>(null);const followRef = useRef(follow);followRef.current = follow;- function follow() {+ async function follow() {pendingRef.current = false;- hasResumedRef.current = false;- refetchSession();+ attemptsLeftRef.current = FOLLOW_ATTEMPTS;+ if (isProbingRef.current) return;+ isProbingRef.current = true;+ try {+ while (attemptsLeftRef.current-- > 0) {+ if (!isMountedRef.current || isBusyRef.current) return;+ hasResumedRef.current = false;+ const result = await refetchSession();+ if (hasActiveBackendStream(result)) return;+ if (attemptsLeftRef.current > 0) await wait(FOLLOW_INTERVAL_MS);+ }+ } finally {+ isProbingRef.current = false;+ }+ }++ function wait(ms: number) {+ return new Promise<void>((wake) => {+ timerRef.current = { id: window.setTimeout(wake, ms), wake };+ });}+ useEffect(() => {+ isMountedRef.current = true;+ return () => {+ isMountedRef.current = false;+ if (timerRef.current) {+ window.clearTimeout(timerRef.current.id);+ timerRef.current.wake();+ }+ };+ }, []);+useEffect(() => {if (!isBusy && pendingRef.current) followRef.current();}, [isBusy]);function followBackendTurn() {pendingRef.current = true;- if (!isBusy) follow();+ if (!isBusyRef.current) followRef.current();}return { followBackendTurn };