AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
[SECRT-2695] fix(platform): stop a retransmitted chat send from queueing and running again (#14939)
Bentlyd90300d5722c
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.pyindex 51df0a5cc6..209d87a987 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py@@ -67,13 +67,19 @@ from backend.copilot.offers import (from backend.copilot.pending_message_helpers import (QueuePendingMessageResponse,StreamRegistryUnavailable,+ already_accepted_response,+ client_message_in_progress_error,is_turn_in_flight,queue_pending_for_http,resolve_attachments_for_http,)from backend.copilot.pending_messages import (+ accept_client_message,+ claim_client_message,clear_pending_messages_unsafe,+ client_message_state,peek_pending_messages,+ release_client_message,)from backend.copilot.provider_failure import ProviderFailure, ProviderFailureKindfrom backend.copilot.provider_tiers import (@@ -317,9 +323,9 @@ class StreamChatRequest(BaseModel):"scopes it to the authenticated user and session before using ""the result as ``ChatMessage.id``. Frontend / network / ""RMQ-redelivery retransmits of the same logical send reuse the "- "key, so the Postgres unique-constraint on the resulting PK is "- "the atomic dedup primitive. A duplicate INSERT returns a "- "subscribe-only response without creating a parallel turn. "+ "key, and the server claims it once: a retransmit neither starts "+ "a parallel turn nor, while a turn is running, queues the "+ "message again, and gets a subscribe-only response. ""Distinct user clicks (even with identical text) MUST send ""different ids — the frontend's per-click ``crypto.randomUUID()`` ""guarantees that."@@ -346,6 +352,15 @@ class QueuePendingMessageRequest(BaseModel):context: dict[str, str] | None = Nonefile_ids: list[str] | None = Field(default=None, max_length=20)folder_ids: list[str] | None = Field(default=None, max_length=5)+ message_id: str | None = Field(+ default=None,+ max_length=64,+ description=(+ "Optional per-send UUID, scoped like ``StreamChatRequest."+ "message_id``. A retransmit of a follow-up already queued is "+ "answered as accepted without queueing it again."+ ),+ )class PeekPendingMessagesResponse(BaseModel):@@ -1839,6 +1854,9 @@ async def stream_chat_post(session_id,)+ # The kickoff owns a server-derived id and its own replay handling.+ client_message_id = None if request.expert_kickoff else message_id+# Session-anchored tenancy: the ChatSession row is the authoritative# org/team for every turn in it — a user whose active header org# differs still charges/attributes turns to the session's org.@@ -1878,6 +1896,7 @@ async def stream_chat_post(file_ids=request.file_ids,folder_ids=request.folder_ids,expert_id=session.expert_id,+ client_message_id=client_message_id,)return _empty_ui_message_stream_response(follow_up_queued=True)except HTTPException as exc:@@ -1977,29 +1996,55 @@ async def stream_chat_post(# near the start) — that path returns early. Any request that# reaches this point is starting a fresh turn, so we always mint a# ``turn_id`` unless ``append_and_save_message`` reports a duplicate.+ #+ # The client key is claimed before the turn can go in flight. From then on+ # a retransmit takes the queue branch above, where the PK cannot catch it:+ # the row may not be written yet, and a queued copy never gets this id.+ # A copy that finds the key reserved waits for this request's outcome, so+ # it is never told a send was taken that is then refused.+ claim_owner = uuid4().hex+ claim = (+ await claim_client_message(session_id, client_message_id, claim_owner)+ if client_message_id is not None+ else None+ )+ if claim == "reserved":+ raise client_message_in_progress_error()++ async def release_client_message_claim() -> None:+ if claim == "claimed" and client_message_id is not None:+ await release_client_message(session_id, client_message_id, claim_owner)++ async def accept_client_message_claim() -> None:+ if claim == "claimed" and client_message_id is not None:+ await accept_client_message(session_id, client_message_id, claim_owner)+try:- turn_id = await schedule_chat_turn(- session_id=session_id,- user_id=user_id,- message=message,- message_id=message_id,- message_metadata=message_metadata,- message_already_persisted=resume_persisted_kickoff,- is_user_message=request.is_user_message,- expert_id=session.expert_id,- session_origin=session.metadata.origin,- session_source_platform=session.metadata.source_platform,- context=request.context,- voice=request.voice,- file_ids=sanitized_file_ids,- organization_id=turn_org_id,- team_id=turn_team_id,- model=request.model,- llm_auth_provider=session.metadata.llm_auth_provider,- llm_credential_id=session.metadata.llm_credential_id,- permissions=builder_permissions,- request_arrival_at=request_arrival_at,- )+ if claim == "accepted":+ turn_id = None+ else:+ turn_id = await schedule_chat_turn(+ session_id=session_id,+ user_id=user_id,+ message=message,+ message_id=message_id,+ message_metadata=message_metadata,+ message_already_persisted=resume_persisted_kickoff,+ is_user_message=request.is_user_message,+ expert_id=session.expert_id,+ session_origin=session.metadata.origin,+ session_source_platform=session.metadata.source_platform,+ context=request.context,+ voice=request.voice,+ file_ids=sanitized_file_ids,+ organization_id=turn_org_id,+ team_id=turn_team_id,+ model=request.model,+ llm_auth_provider=session.metadata.llm_auth_provider,+ llm_credential_id=session.metadata.llm_credential_id,+ permissions=builder_permissions,+ request_arrival_at=request_arrival_at,+ )except ConcurrentTurnLimitError as exc:if resume_persisted_kickoff:raise HTTPException(status_code=429, detail=str(exc)) from exc@@ -2031,15 +2076,26 @@ async def stream_chat_post(request_arrival_at=request_arrival_at,)except turn_queue.InflightCapExceeded:+ await release_client_message_claim()raise HTTPException(status_code=429,detail=inflight_turn_limit_message(inflight_cap),)+ except BaseException:+ # Raised inside the handler above, so the outer BaseException+ # clause never sees it.+ await release_client_message_claim()+ raise+ await accept_client_message_claim()logger.info(f"[STREAM] Queued turn for session={session_id} "f"(running cap reached; inflight cap={inflight_cap})")return _empty_ui_message_stream_response()+ except BaseException:+ await release_client_message_claim()+ raise+ await accept_client_message_claim()if turn_id is None:logger.info(@@ -2244,6 +2300,19 @@ async def queue_pending_message(session = await _validate_and_get_writable_session(session_id, user_id)if session.metadata.llm_auth_provider == "codex":await enforce_codex_access_http(user_id)+ client_message_id = (+ scoped_client_message_id(user_id, session_id, request.message_id)+ if request.message_id+ else None+ )+ # Before the in-flight gate: a retransmit landing after the turn drained+ # its original must not 409, or the client re-sends it via POST /stream.+ if client_message_id is not None:+ state = await client_message_state(session_id, client_message_id)+ if state == "accepted":+ return await already_accepted_response(session_id)+ if state == "reserved":+ raise client_message_in_progress_error()try:turn_in_flight = await is_turn_in_flight(session_id)except StreamRegistryUnavailable as exc:@@ -2265,6 +2334,7 @@ async def queue_pending_message(file_ids=request.file_ids,folder_ids=request.folder_ids,expert_id=session.expert_id,+ client_message_id=client_message_id,)diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.pyindex f9f3ed6a6f..4ab3d61c54 100644--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMockimport fastapiimport fastapi.testclient+import httpximport pytestimport pytest_mockfrom pydantic import SecretStr@@ -15,6 +16,7 @@ from sentry_sdk.integrations.asgi import SentryAsgiMiddlewarefrom backend.api.features.chat import routes as chat_routesfrom backend.api.features.chat.routes import _strip_injected_context+from backend.copilot import pending_messages as pending_messages_modulefrom backend.copilot import transports as chat_transportsfrom backend.copilot.config import CopilotLlmAuthProviderfrom backend.copilot.model import ChatSession@@ -347,11 +349,13 @@ def _mock_stream_internals(new_callable=AsyncMock,return_value=None,)+ claims = _mock_client_message_claims(mocker)return types.SimpleNamespace(enqueue=mock_schedule,paywall=mock_paywall,session=mock_session,owns_active_expert=mock_owns_active_expert,+ claims=claims,)@@ -741,6 +745,281 @@ def test_stream_chat_scopes_client_message_id_to_owner_and_session()+# ─── Client message id idempotency (SECRT-2695) ───────────────────────+++class _ClaimRedis:+ """The client-message claim's SET NX / GET / compare-and-set, plus the+ LLEN a duplicate's response reads the buffer length with."""++ def __init__(self) -> None:+ self.values: dict[str, str] = {}++ @property+ def keys(self) -> set[str]:+ return set(self.values)++ async def set(+ self, key: str, value: str, nx: bool = False, ex: int | None = None+ ) -> bool | None:+ if nx and key in self.values:+ return None+ self.values[key] = value+ return True++ async def get(self, key: str) -> str | None:+ return self.values.get(key)++ async def eval(+ self,+ script: str,+ numkeys: int,+ key: str,+ expected: str,+ new: str,+ ttl_seconds: int,+ or_missing: str,+ ) -> int:+ """``string_compare_and_set``, which settles a claim."""+ current = self.values.get(key)+ if current != expected and not (current is None and or_missing == "1"):+ return 0+ if new:+ self.values[key] = new+ else:+ self.values.pop(key, None)+ return 1++ async def llen(self, key: str) -> int:+ return 0+++def _mock_client_message_claims(mocker: pytest_mock.MockerFixture) -> _ClaimRedis:+ redis = _ClaimRedis()+ mocker.patch(+ "backend.copilot.pending_messages.get_redis_async",+ new_callable=AsyncMock,+ return_value=redis,+ )+ return redis+++def _mock_pending_push(mocker: pytest_mock.MockerFixture) -> AsyncMock:+ """Run the real ``queue_pending_for_http`` down to the Redis push."""+ mocker.patch(+ "backend.copilot.pending_message_helpers.get_redis_async",+ new_callable=AsyncMock,+ return_value=mocker.MagicMock(),+ )+ mocker.patch(+ "backend.copilot.pending_message_helpers.incr_with_ttl",+ new_callable=AsyncMock,+ return_value=1,+ )+ return mocker.patch(+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",+ new_callable=AsyncMock,+ return_value=1,+ )+++def test_stream_chat_retransmits_of_a_started_turn_queue_nothing(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """The same send arriving three times starts one turn. The copies that+ land once the turn is running used to be buffered as follow-ups, each+ one becoming another user message and another turn."""+ mocks = _mock_stream_internals(mocker)+ push = _mock_pending_push(mocker)+ in_flight = mocker.patch(+ "backend.api.features.chat.routes.is_turn_in_flight",+ new_callable=AsyncMock,+ return_value=False,+ )+ body = {"message": "hello", "message_id": "client-click-id"}++ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200+ in_flight.return_value = True+ for _ in range(2):+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200++ mocks.enqueue.assert_awaited_once()+ push.assert_not_awaited()+++def test_stream_chat_retransmits_of_a_queued_send_are_buffered_once(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """A stale tab posts to /stream while another tab's turn runs, so the+ send is queued. Its copies must neither queue again nor, once the turn+ has drained it, start a turn of their own."""+ mocks = _mock_stream_internals(mocker)+ push = _mock_pending_push(mocker)+ in_flight = mocker.patch(+ "backend.api.features.chat.routes.is_turn_in_flight",+ new_callable=AsyncMock,+ return_value=True,+ )+ body = {"message": "hello", "message_id": "client-click-id"}++ for _ in range(3):+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200+ in_flight.return_value = False+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200++ push.assert_awaited_once()+ mocks.enqueue.assert_not_awaited()+++def test_stream_chat_distinct_message_ids_are_each_queued(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """Two clicks with the same text are two messages."""+ _mock_stream_internals(mocker)+ push = _mock_pending_push(mocker)+ mocker.patch(+ "backend.api.features.chat.routes.is_turn_in_flight",+ new_callable=AsyncMock,+ return_value=True,+ )++ for message_id in ("click-1", "click-2"):+ response = client.post(+ "/sessions/sess-1/stream",+ json={"message": "hello", "message_id": message_id},+ )+ assert response.status_code == 200++ assert push.await_count == 2+++def test_stream_chat_refused_send_can_be_retried_with_the_same_id(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """A send the server turned away was never accepted, so its id must not+ stay claimed: the retry would be dropped as a duplicate of nothing."""+ mocks = _mock_stream_internals(mocker)+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")+ mocker.patch.object(+ chat_routes.turn_queue,+ "try_enqueue_turn",+ new_callable=AsyncMock,+ side_effect=chat_routes.turn_queue.InflightCapExceeded(),+ )+ body = {"message": "hello", "message_id": "client-click-id"}++ assert client.post("/sessions/sess-1/stream", json=body).status_code == 429+ assert mocks.claims.keys == set()++ mocks.enqueue.side_effect = None+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200+ assert mocks.enqueue.await_count == 2+++def test_stream_chat_failed_queue_fallback_can_be_retried_with_the_same_id(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """The running cap sends the turn to the queue. If that fails too, the+ send was never accepted and its id must be free for the retry."""+ mocks = _mock_stream_internals(mocker)+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")+ mocker.patch.object(+ chat_routes.turn_queue,+ "try_enqueue_turn",+ new_callable=AsyncMock,+ side_effect=RuntimeError("database unavailable"),+ )+ body = {"message": "hello", "message_id": "client-click-id"}+ safe_client = fastapi.testclient.TestClient(app, raise_server_exceptions=False)++ response = safe_client.post("/sessions/sess-1/stream", json=body)+ assert response.status_code == 500+ assert mocks.claims.keys == set()++ mocks.enqueue.side_effect = None+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200+ assert mocks.enqueue.await_count == 2+++async def test_stream_chat_copy_carries_a_send_the_first_copy_had_refused(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """A copy lands while the first copy is being scheduled, and the first is+ then refused at the in-flight cap. The send was never taken, so the copy+ must schedule it, not be answered as a duplicate of nothing."""+ mocks = _mock_stream_internals(mocker)+ first_scheduling = asyncio.Event()+ let_first_finish = asyncio.Event()+ scheduled: list[str] = []++ async def schedule(**kwargs: object) -> str:+ scheduled.append("call")+ if len(scheduled) == 1:+ first_scheduling.set()+ await let_first_finish.wait()+ raise chat_routes.ConcurrentTurnLimitError("busy")+ return "turn-2"++ mocks.enqueue.side_effect = schedule+ mocker.patch.object(+ chat_routes.turn_queue,+ "try_enqueue_turn",+ new_callable=AsyncMock,+ side_effect=chat_routes.turn_queue.InflightCapExceeded(),+ )+ body = {"message": "hello", "message_id": "client-click-id"}++ async with httpx.AsyncClient(+ transport=httpx.ASGITransport(app=app), base_url="http://test"+ ) as async_client:+ first = asyncio.create_task(+ async_client.post("/sessions/sess-1/stream", json=body)+ )+ await first_scheduling.wait()+ copy = asyncio.create_task(+ async_client.post("/sessions/sess-1/stream", json=body)+ )+ await asyncio.sleep(0.2)+ let_first_finish.set()+ first_response = await first+ copy_response = await copy++ assert first_response.status_code == 429+ assert copy_response.status_code == 200+ assert len(scheduled) == 2+ assert list(mocks.claims.values.values()) == ["accepted"]+++def _reserve_client_message(claims: _ClaimRedis, user_id: str) -> None:+ """Hold the id the way a first copy still being scheduled does."""+ scoped_id = chat_routes.scoped_client_message_id(+ user_id, "sess-1", "client-click-id"+ )+ key = pending_messages_module._client_message_key("sess-1", scoped_id)+ claims.values[key] = "reserved:first-copy"+++def test_stream_chat_copy_is_told_to_retry_while_the_first_is_unsettled(+ mocker: pytest_mock.MockerFixture,+ test_user_id: str,+) -> None:+ """The first copy is still being scheduled and may yet be refused, so+ this one is told neither that the send was taken nor that it failed."""+ mocks = _mock_stream_internals(mocker)+ mocker.patch.object(pending_messages_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)+ _reserve_client_message(mocks.claims, test_user_id)++ response = client.post(+ "/sessions/sess-1/stream",+ json={"message": "hello", "message_id": "client-click-id"},+ )++ assert response.status_code == 503+ mocks.enqueue.assert_not_awaited()+ assert list(mocks.claims.values.values()) == ["reserved:first-copy"]++# ─── UUID format filtering ─────────────────────────────────────────────@@ -2379,6 +2658,60 @@ def test_queue_pending_message_returns_200_when_turn_in_flight(assert "turn_in_flight" in data+def test_queue_pending_message_same_message_id_is_buffered_once(+ mocker: pytest_mock.MockerFixture,+) -> None:+ """A retransmitted follow-up is queued once, and a copy that lands after+ the turn ended is answered as accepted rather than 409 (which would send+ the client to POST /stream with the message all over again)."""+ _mock_stream_queue_internals(mocker)+ _mock_client_message_claims(mocker)+ push = mocker.patch(+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",+ new_callable=AsyncMock,+ return_value=1,+ )+ body = {"message": "follow-up", "message_id": "client-click-id"}++ for _ in range(2):+ response = client.post("/sessions/sess-1/messages/pending", json=body)+ assert response.status_code == 200+ assert response.json()["turn_in_flight"] is True+ mocker.patch(+ "backend.api.features.chat.routes.is_turn_in_flight",+ new_callable=AsyncMock,+ return_value=False,+ )+ response = client.post("/sessions/sess-1/messages/pending", json=body)++ assert response.status_code == 200+ assert response.json()["turn_in_flight"] is True+ push.assert_awaited_once()+++def test_queue_pending_message_copy_is_told_to_retry_while_the_first_is_unsettled(+ mocker: pytest_mock.MockerFixture,+ test_user_id: str,+) -> None:+ _mock_stream_queue_internals(mocker)+ claims = _mock_client_message_claims(mocker)+ mocker.patch.object(pending_messages_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)+ _reserve_client_message(claims, test_user_id)+ push = mocker.patch(+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",+ new_callable=AsyncMock,+ return_value=1,+ )++ response = client.post(+ "/sessions/sess-1/messages/pending",+ json={"message": "follow-up", "message_id": "client-click-id"},+ )++ assert response.status_code == 503+ push.assert_not_awaited()++def test_queue_pending_message_session_not_found_returns_404(mocker: pytest_mock.MockerFixture,) -> None:diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers.pyindex 52f9f6faa4..ce540b56a5 100644--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers.py@@ -7,7 +7,9 @@ Also provides the call-rate-limit check for the queue endpoint soroutes.py stays free of Redis/Lua details."""+import asyncioimport logging+import uuidfrom typing import TYPE_CHECKING, Any, Callablefrom fastapi import HTTPException@@ -24,10 +26,15 @@ from backend.copilot.pending_messages import (MAX_PENDING_MESSAGES,PendingMessage,PendingMessageContext,+ accept_client_message,+ claim_client_message,drain_pending_messages,+ finish_despite_cancellation,format_pending_as_user_message,+ peek_pending_count,push_pending_message,push_pending_message_if_session_running,+ release_client_message,)from backend.copilot.response_model import (StreamPendingDrained,@@ -211,6 +218,7 @@ async def queue_pending_for_http(file_ids: list[str] | None,folder_ids: list[str] | None,expert_id: str | None,+ client_message_id: str | None = None,) -> QueuePendingMessageResponse:"""HTTP-facing wrapper around :func:`queue_user_message`.@@ -226,6 +234,13 @@ async def queue_pending_for_http(as a field: a pending message is rendered from ``content`` when the turndrains it, and entries written by older workers are still in Redis.+ ``client_message_id`` is the send's scoped idempotency key. A send+ whose key was already accepted is answered without pushing again (see+ :func:`already_accepted_response`), and a push that does not land gives+ the key back so a genuine retry can. A copy that arrives while another+ is still being pushed waits for its outcome, and gets a 503 if that+ outlasts the wait.+Raises :class:`HTTPException` with status 429 if the rate cap is hit or400 if an expert session attaches a file outside its scope; otherwisereturns the ``QueuePendingMessageResponse`` the handler can serialise 1:1.@@ -256,18 +271,56 @@ async def queue_pending_for_http(# the FE's is_turn_in_flight check and our gate), which both this# endpoint and the POST /stream queue-fall-through can hit. Pushing# first lets the gate own the no-op short-circuit.- response = await queue_user_message(- session_id=session_id,- message=message,- context=queue_context,- file_ids=sanitized_file_ids,- require_turn_in_flight=True,+ claim_owner = uuid.uuid4().hex+ if client_message_id is not None:+ claim = await claim_client_message(session_id, client_message_id, claim_owner)+ if claim == "accepted":+ logger.info(+ "pending_messages: skipped retransmit of an accepted message "+ "for session=%s",+ session_id,+ )+ return await already_accepted_response(session_id)+ if claim == "reserved":+ raise client_message_in_progress_error()++ push = asyncio.ensure_future(+ queue_user_message(+ session_id=session_id,+ message=message,+ context=queue_context,+ file_ids=sanitized_file_ids,+ require_turn_in_flight=True,+ ))+ try:+ response = await asyncio.shield(push)+ except asyncio.CancelledError:+ # The append may already have landed (the push still awaits its+ # SPUBLISH), so settle the claim by what the push did, not by the+ # cancellation: releasing it would let a retry queue a second copy.+ if client_message_id is not None:+ await finish_despite_cancellation(+ asyncio.ensure_future(+ _settle_claim_of_cancelled_push(+ push, session_id, client_message_id, claim_owner+ )+ )+ )+ raise+ except BaseException:+ if client_message_id is not None:+ await release_client_message(session_id, client_message_id, claim_owner)+ raiseif not response.turn_in_flight:+ if client_message_id is not None:+ await release_client_message(session_id, client_message_id, claim_owner)raise HTTPException(status_code=409,detail="Session has no active turn. Start a new turn with POST /stream.",)+ if client_message_id is not None:+ await accept_client_message(session_id, client_message_id, claim_owner)# Push landed — now charge the rate counter. If this tick crosses the# limit we still keep the queued message (next drain will pick it up)@@ -286,6 +339,60 @@ async def queue_pending_for_http(return response+async def _settle_claim_of_cancelled_push(+ push: "asyncio.Future[QueuePendingMessageResponse]",+ session_id: str,+ client_message_id: str,+ owner: str,+) -> None:+ try:+ response = await push+ except Exception:+ await release_client_message(session_id, client_message_id, owner)+ return+ if response.turn_in_flight:+ await accept_client_message(session_id, client_message_id, owner)+ else:+ await release_client_message(session_id, client_message_id, owner)+++def client_message_in_progress_error() -> HTTPException:+ """For a copy that arrived while the first copy was still being scheduled+ and outlasted the wait for its outcome. It is neither accepted nor+ refused yet, so the client is told to retry, as for a degraded registry.+ """+ return HTTPException(+ status_code=503,+ detail="Your message is still being sent, retry shortly",+ headers={"Retry-After": "5"},+ )+++async def already_accepted_response(session_id: str) -> QueuePendingMessageResponse:+ """Answer a retransmit of a send the server already accepted.++ ``turn_in_flight`` is ``True`` whatever the turn is doing now: it tells the+ client its message was taken, and ``False`` would make it fall back to+ ``POST /stream`` and send the message all over again.+ """+ try:+ buffer_length = await peek_pending_count(session_id)+ except Exception as e:+ # The count is informational. A Redis blip must not turn an accepted+ # send into an error, which the client would answer by resending it.+ logger.warning(+ "pending_messages: buffer length lookup failed for session=%s: %s",+ session_id,+ e,+ )+ buffer_length = 0+ return QueuePendingMessageResponse(+ buffer_length=buffer_length,+ max_buffer_length=MAX_PENDING_MESSAGES,+ turn_in_flight=True,+ )++async def check_pending_call_rate(user_id: str) -> int:"""Increment and return the per-user push counter for the current window.diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.pyindex ab9e0a7b30..95a9afc8f3 100644--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py@@ -1,8 +1,9 @@"""Unit tests for pending_message_helpers."""+import asynciofrom types import SimpleNamespacefrom typing import Any-from unittest.mock import AsyncMock, MagicMock+from unittest.mock import ANY, AsyncMock, MagicMockimport pytestfrom fastapi import HTTPException@@ -284,6 +285,241 @@ async def test_queue_pending_429_after_push_when_limit_exceeded(queue_mock.assert_awaited_once()+# ── queue_pending_for_http: client message id (SECRT-2695) ──────────+++def _mock_queue_claims(+ monkeypatch: pytest.MonkeyPatch, *, claim_result: str+) -> tuple[AsyncMock, AsyncMock, AsyncMock]:+ claim = AsyncMock(return_value=claim_result)+ accept = AsyncMock()+ release = AsyncMock()+ monkeypatch.setattr(helpers_module, "claim_client_message", claim)+ monkeypatch.setattr(helpers_module, "accept_client_message", accept)+ monkeypatch.setattr(helpers_module, "release_client_message", release)+ monkeypatch.setattr(helpers_module, "peek_pending_count", AsyncMock(return_value=1))+ return claim, accept, release+++async def _queue_with_client_id() -> QueuePendingMessageResponse:+ return await queue_pending_for_http(+ session_id="sess-1",+ user_id="user-1",+ message="hi",+ context=None,+ file_ids=None,+ folder_ids=None,+ expert_id=None,+ client_message_id="scoped-msg-1",+ )+++async def test_queue_pending_skips_retransmit_of_an_accepted_message(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A retransmit is answered as accepted without a second push or a+ rate tick, and says the turn is in flight so the client does not fall+ back to POST /stream."""+ _mock_queue_claims(monkeypatch, claim_result="accepted")+ queue_mock = AsyncMock()+ monkeypatch.setattr(helpers_module, "queue_user_message", queue_mock)+ rate_mock = AsyncMock(return_value=1)+ monkeypatch.setattr(helpers_module, "check_pending_call_rate", rate_mock)++ result = await _queue_with_client_id()++ assert result.turn_in_flight is True+ assert result.buffer_length == 1+ queue_mock.assert_not_awaited()+ rate_mock.assert_not_awaited()+++async def test_queue_pending_keeps_the_claim_once_the_push_lands(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ claim, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")+ monkeypatch.setattr(+ helpers_module,+ "queue_user_message",+ AsyncMock(+ return_value=QueuePendingMessageResponse(+ buffer_length=1,+ max_buffer_length=MAX_PENDING_MESSAGES,+ turn_in_flight=True,+ )+ ),+ )+ monkeypatch.setattr(+ helpers_module, "check_pending_call_rate", AsyncMock(return_value=1)+ )++ await _queue_with_client_id()++ claim.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ release.assert_not_awaited()+++async def test_queue_pending_releases_the_claim_when_the_push_is_refused(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The turn ended before the push: the route falls through to start a+ turn with the same id, which needs the claim back."""+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")+ monkeypatch.setattr(+ helpers_module,+ "queue_user_message",+ AsyncMock(+ return_value=QueuePendingMessageResponse(+ buffer_length=0,+ max_buffer_length=MAX_PENDING_MESSAGES,+ turn_in_flight=False,+ )+ ),+ )++ with pytest.raises(HTTPException) as exc_info:+ await _queue_with_client_id()++ assert exc_info.value.status_code == 409+ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ accept.assert_not_awaited()+++async def test_queue_pending_releases_the_claim_when_the_push_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")+ monkeypatch.setattr(+ helpers_module,+ "queue_user_message",+ AsyncMock(side_effect=ConnectionError("down")),+ )++ with pytest.raises(ConnectionError):+ await _queue_with_client_id()++ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ accept.assert_not_awaited()++[email protected]("landed", [True, False])+async def test_queue_pending_settles_the_claim_by_the_push_when_cancelled(+ monkeypatch: pytest.MonkeyPatch, landed: bool+) -> None:+ """The request is cancelled after the append, while the push still awaits+ its SPUBLISH: the push finishes and the claim follows its real outcome,+ so a retry of a message that is already queued is not pushed again."""+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")+ appended = asyncio.Event()+ published = asyncio.Event()++ async def push(**_: Any) -> QueuePendingMessageResponse:+ appended.set()+ await published.wait()+ return QueuePendingMessageResponse(+ buffer_length=1 if landed else 0,+ max_buffer_length=MAX_PENDING_MESSAGES,+ turn_in_flight=landed,+ )++ monkeypatch.setattr(helpers_module, "queue_user_message", push)+ request = asyncio.create_task(_queue_with_client_id())+ await appended.wait()+ request.cancel()+ await asyncio.sleep(0)+ published.set()++ with pytest.raises(asyncio.CancelledError):+ await request++ if landed:+ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ release.assert_not_awaited()+ else:+ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ accept.assert_not_awaited()+++async def test_queue_pending_settles_the_claim_when_cancelled_again_while_settling(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """A second cancellation while the claim is being settled must not cut the+ push or the settle short, or the reservation is left to lapse."""+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")+ appended = asyncio.Event()+ published = asyncio.Event()++ async def push(**_: Any) -> QueuePendingMessageResponse:+ appended.set()+ await published.wait()+ return QueuePendingMessageResponse(+ buffer_length=1,+ max_buffer_length=MAX_PENDING_MESSAGES,+ turn_in_flight=True,+ )++ monkeypatch.setattr(helpers_module, "queue_user_message", push)+ request = asyncio.create_task(_queue_with_client_id())+ await appended.wait()+ request.cancel()+ await asyncio.sleep(0)+ await asyncio.sleep(0)+ request.cancel()+ await asyncio.sleep(0)+ published.set()++ with pytest.raises(asyncio.CancelledError):+ await request++ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)+ release.assert_not_awaited()+++async def test_queue_pending_tells_a_copy_to_retry_while_the_first_is_unsettled(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The first copy is still being pushed, so this one can be told neither+ that the message was taken nor that it was refused."""+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="reserved")+ queue_mock = AsyncMock()+ monkeypatch.setattr(helpers_module, "queue_user_message", queue_mock)++ with pytest.raises(HTTPException) as exc_info:+ await _queue_with_client_id()++ assert exc_info.value.status_code == 503+ queue_mock.assert_not_awaited()+ accept.assert_not_awaited()+ release.assert_not_awaited()+++async def test_queue_pending_retransmit_is_accepted_when_the_count_lookup_fails(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """The buffer length is informational, so a Redis error reading it still+ answers the copy as accepted rather than failing it."""+ _mock_queue_claims(monkeypatch, claim_result="accepted")+ monkeypatch.setattr(+ helpers_module,+ "peek_pending_count",+ AsyncMock(side_effect=ConnectionError("down")),+ )++ result = await _queue_with_client_id()++ assert result.turn_in_flight is True+ assert result.buffer_length == 0++@pytest.mark.asyncioasync def test_check_pending_call_rate_at_limit(monkeypatch: pytest.MonkeyPatch,diff --git a/autogpt_platform/backend/backend/copilot/pending_messages.py b/autogpt_platform/backend/backend/copilot/pending_messages.pyindex e6e0274ef8..3fe1e4b7dd 100644--- a/autogpt_platform/backend/backend/copilot/pending_messages.py+++ b/autogpt_platform/backend/backend/copilot/pending_messages.py@@ -21,19 +21,26 @@ A hard cap of ``MAX_PENDING_MESSAGES`` per session prevents abuse. Thebuffer is trimmed to the latest ``MAX_PENDING_MESSAGES`` on every push."""+import asyncioimport jsonimport loggingimport timeimport uuid-from typing import Any, cast+from typing import Any, Literal, TypeVar, castfrom pydantic import BaseModel, Field, ValidationErrorfrom backend.data.redis_client import get_redis_async-from backend.data.redis_helpers import capped_rpush, capped_rpush_if_hash_field+from backend.data.redis_helpers import (+ capped_rpush,+ capped_rpush_if_hash_field,+ string_compare_and_set,+)logger = logging.getLogger(__name__)+T = TypeVar("T")+# Per-session cap; typing faster than the copilot drains is already unusual.MAX_PENDING_MESSAGES = 10@@ -51,6 +58,24 @@ _PERSIST_QUEUE_KEY_PREFIX = "copilot:pending-persist:"# message as a wake-up hint; the value itself is not meaningful._NOTIFY_PAYLOAD = "1"+# Client idempotency keys (``StreamChatRequest.message_id``, already scoped to+# user + session) of sends the chat API has accepted. A buffered message has+# no ``ChatMessage`` row until a turn drains it, and the drained row gets a+# fresh id, so the PK that dedupes a turn-start send cannot catch a+# retransmit of a queued one: without this claim every copy was buffered+# again and ran as another follow-up.+_CLIENT_MESSAGE_KEY_PREFIX = "copilot:pending:client-msg:"+# A reservation names the request holding it, so one that outlived its TTL+# cannot accept or release the claim of the request that took over.+_CLIENT_MESSAGE_RESERVED = "reserved:"+_CLIENT_MESSAGE_ACCEPTED = "accepted"+# A reservation covers scheduling a turn or pushing to the buffer, which takes+# well under a second; the TTL only matters when the request dies midway.+_CLIENT_MESSAGE_RESERVE_TTL_SECONDS = 30+# How long a copy waits for the first copy's outcome before it gives up.+_CLIENT_MESSAGE_WAIT_SECONDS = 10.0+_CLIENT_MESSAGE_POLL_SECONDS = 0.1+class PendingMessageContext(BaseModel):"""Structured page context attached to a pending message.@@ -94,6 +119,165 @@ def _notify_channel(session_id: str) -> str:return f"{_PENDING_CHANNEL_PREFIX}{session_id}"+def _client_message_key(session_id: str, message_id: str) -> str:+ return f"{_CLIENT_MESSAGE_KEY_PREFIX}{session_id}:{message_id}"+++ClientMessageClaim = Literal["claimed", "accepted", "reserved"]+++async def _settled_client_message(redis: Any, key: str, deadline: float) -> str | None:+ """The key's state once no other request is still scheduling its send,+ or ``"reserved"`` if that has not happened by *deadline*."""+ while True:+ state = await redis.get(key)+ if not _is_reservation(state) or time.monotonic() >= deadline:+ return state+ await asyncio.sleep(_CLIENT_MESSAGE_POLL_SECONDS)+++def _is_reservation(state: str | None) -> bool:+ return state is not None and state.startswith(_CLIENT_MESSAGE_RESERVED)+++async def claim_client_message(+ session_id: str, message_id: str, owner: str+) -> ClientMessageClaim:+ """Reserve *message_id* for the request carrying it, identified by the+ unique *owner* token it passes to accept or release the claim.++ ``"claimed"``: the caller owns the send. It must call+ :func:`accept_client_message` once the send is scheduled or queued, or+ :func:`release_client_message` when it is refused. The reservation+ expires on its own within seconds, so a request that dies midway+ cannot block the id.++ ``"accepted"``: the caller holds a retransmit of a message that is+ queued, drained or persisted, and must not act on it again.++ ``"reserved"``: another copy was still being scheduled when the wait ran+ out, so the caller should tell the client to retry. A copy waits for+ that outcome rather than being told it was accepted: the first copy can+ still be refused, and then the send was never taken.++ Fails open to ``"claimed"`` on a Redis error so dedup can never block a+ send.+ """+ key = _client_message_key(session_id, message_id)+ deadline = time.monotonic() + _CLIENT_MESSAGE_WAIT_SECONDS+ try:+ redis = await get_redis_async()+ while True:+ if await redis.set(+ key,+ _CLIENT_MESSAGE_RESERVED + owner,+ nx=True,+ ex=_CLIENT_MESSAGE_RESERVE_TTL_SECONDS,+ ):+ return "claimed"+ state = await _settled_client_message(redis, key, deadline)+ if _is_reservation(state):+ return "reserved"+ if state is not None:+ return "accepted"+ # The first copy was refused and gave the id back: take it.+ except Exception as e:+ logger.warning(+ "pending_messages: client message claim failed for session=%s: %s",+ session_id,+ e,+ )+ return "claimed"+++async def finish_despite_cancellation(work: "asyncio.Future[T]") -> T:+ """Wait for shielded cleanup *work* however many times the caller is+ cancelled meanwhile; the caller re-raises its own cancellation after."""+ while True:+ try:+ return await asyncio.shield(work)+ except asyncio.CancelledError:+ if work.done():+ return work.result()+++async def accept_client_message(session_id: str, message_id: str, owner: str) -> None:+ """Mark a claimed send as taken, so its retransmits are skipped. Also+ when the reservation has lapsed, unless another request holds it now.++ Finishes even if the caller is cancelled midway: the send has already+ landed, and a reservation left to lapse would let a retry send it again.+ """+ accept = asyncio.ensure_future(+ _accept_client_message(session_id, message_id, owner)+ )+ try:+ await asyncio.shield(accept)+ except asyncio.CancelledError:+ await finish_despite_cancellation(accept)+ raise+++async def _accept_client_message(session_id: str, message_id: str, owner: str) -> None:+ try:+ redis = await get_redis_async()+ await string_compare_and_set(+ redis,+ _client_message_key(session_id, message_id),+ expected=_CLIENT_MESSAGE_RESERVED + owner,+ new=_CLIENT_MESSAGE_ACCEPTED,+ ttl_seconds=_PENDING_TTL_SECONDS,+ or_missing=True,+ )+ except Exception as e:+ logger.warning(+ "pending_messages: client message accept failed for session=%s: %s",+ session_id,+ e,+ )+++async def client_message_state(+ session_id: str, message_id: str+) -> Literal["accepted", "reserved"] | None:+ """Whether a send carrying *message_id* was taken, waiting as+ :func:`claim_client_message` does while another copy is scheduling it."""+ deadline = time.monotonic() + _CLIENT_MESSAGE_WAIT_SECONDS+ try:+ redis = await get_redis_async()+ state = await _settled_client_message(+ redis, _client_message_key(session_id, message_id), deadline+ )+ except Exception as e:+ logger.warning(+ "pending_messages: client message lookup failed for session=%s: %s",+ session_id,+ e,+ )+ return None+ if state is None:+ return None+ return "reserved" if _is_reservation(state) else "accepted"+++async def release_client_message(session_id: str, message_id: str, owner: str) -> None:+ """Drop a claim whose send was refused, so a genuine retry can land."""+ try:+ redis = await get_redis_async()+ await string_compare_and_set(+ redis,+ _client_message_key(session_id, message_id),+ expected=_CLIENT_MESSAGE_RESERVED + owner,+ new=None,+ )+ except Exception as e:+ logger.warning(+ "pending_messages: client message release failed for session=%s: %s",+ session_id,+ e,+ )++def _decode_redis_item(item: Any) -> str:"""Decode a redis-py list item to str (handles ``bytes`` and ``str``)."""return item.decode("utf-8") if isinstance(item, bytes) else str(item)diff --git a/autogpt_platform/backend/backend/copilot/pending_messages_test.py b/autogpt_platform/backend/backend/copilot/pending_messages_test.pyindex 16f03f1f97..5fb87573dc 100644--- a/autogpt_platform/backend/backend/copilot/pending_messages_test.py+++ b/autogpt_platform/backend/backend/copilot/pending_messages_test.py@@ -8,6 +8,7 @@ and pull in the full app startup).import asyncioimport jsonfrom typing import Any+from unittest.mock import AsyncMockimport pytest@@ -16,7 +17,10 @@ from backend.copilot.pending_messages import (MAX_PENDING_MESSAGES,PendingMessage,PendingMessageContext,+ accept_client_message,+ claim_client_message,clear_pending_messages_unsafe,+ client_message_state,drain_and_format_for_injection,drain_pending_for_persist,drain_pending_messages,@@ -25,6 +29,7 @@ from backend.copilot.pending_messages import (peek_pending_count,peek_pending_messages,push_pending_message,+ release_client_message,stash_pending_for_persist,)@@ -37,6 +42,8 @@ class _FakeRedis:# bytes when ``decode_responses=False``; the drain path must# handle both and our tests exercise both.self.lists: dict[str, list[str | bytes]] = {}+ self.strings: dict[str, str] = {}+ self.ttls: dict[str, int | None] = {}self.published: list[tuple[str, str]] = []async def rpush(self, key: str, *values: Any) -> int:@@ -92,7 +99,43 @@ class _FakeRedis:if key in self.lists:del self.lists[key]return 1- return 0+ return int(self.strings.pop(key, None) is not None)++ async def set(+ self, key: str, value: str, nx: bool = False, ex: int | None = None+ ) -> bool | None:+ if nx and key in self.strings:+ return None+ self.strings[key] = value+ self.ttls[key] = ex+ return True++ async def get(self, key: str) -> str | None:+ return self.strings.get(key)++ async def eval(+ self,+ script: str,+ numkeys: int,+ key: str,+ expected: str,+ new: str,+ ttl_seconds: int,+ or_missing: str,+ ) -> int:+ """``string_compare_and_set``, the only script these tests reach."""+ current = self.strings.get(key)+ if current != expected and not (current is None and or_missing == "1"):+ return 0+ if new:+ self.strings[key] = new+ self.ttls[key] = int(ttl_seconds)+ else:+ self.strings.pop(key, None)+ return 1++ async def exists(self, key: str) -> int:+ return int(key in self.strings or key in self.lists)def pipeline(self, transaction: bool = True) -> "_FakePipeline":# Returns a fake pipeline that records ops and replays them in@@ -716,3 +759,203 @@ def test_buffer_and_session_meta_keys_share_cluster_slot() -> None:f"CROSSSLOT regression: {buf!r} (slot {_redis_keyslot(buf)}) "f"!= {meta!r} (slot {_redis_keyslot(meta)})")+++# ── Client message claims (SECRT-2695) ──────────────────────────────+++def short_claim_wait(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(pm_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)+ monkeypatch.setattr(pm_module, "_CLIENT_MESSAGE_POLL_SECONDS", 0.01)+++async def test_client_message_claim_is_granted_once(fake_redis: _FakeRedis) -> None:+ """A retransmit of an accepted send must find the id already taken."""+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ await accept_client_message("sess-1", "msg-a", "first")+ assert await claim_client_message("sess-1", "msg-a", "copy") == "accepted"+ assert await client_message_state("sess-1", "msg-a") == "accepted"+++async def test_client_message_reservation_is_short_and_acceptance_long(+ fake_redis: _FakeRedis,+) -> None:+ """A request that dies before settling its send must not block the id+ for the hour an accepted send is remembered."""+ key = pm_module._client_message_key("sess-1", "msg-a")++ await claim_client_message("sess-1", "msg-a", "first")+ assert fake_redis.ttls[key] == pm_module._CLIENT_MESSAGE_RESERVE_TTL_SECONDS+ await accept_client_message("sess-1", "msg-a", "first")+ assert fake_redis.ttls[key] == pm_module._PENDING_TTL_SECONDS+++async def test_client_message_claims_are_per_id_and_session(+ fake_redis: _FakeRedis,+) -> None:+ assert await claim_client_message("sess-1", "msg-a", "o1") == "claimed"+ assert await claim_client_message("sess-1", "msg-b", "o2") == "claimed"+ assert await claim_client_message("sess-2", "msg-a", "o3") == "claimed"+++async def test_released_client_message_can_be_claimed_again(+ fake_redis: _FakeRedis,+) -> None:+ """A refused send drops its claim so the same id can land on retry."""+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ await release_client_message("sess-1", "msg-a", "first")+ assert await client_message_state("sess-1", "msg-a") is None+ assert await claim_client_message("sess-1", "msg-a", "retry") == "claimed"+++async def test_lapsed_claimant_cannot_settle_its_successors_claim(+ fake_redis: _FakeRedis,+) -> None:+ """The first request outlived its reservation and another took the id+ over: the first can neither release nor accept the successor's claim."""+ key = pm_module._client_message_key("sess-1", "msg-a")+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ del fake_redis.strings[key] # the reservation expired+ assert await claim_client_message("sess-1", "msg-a", "successor") == "claimed"++ await release_client_message("sess-1", "msg-a", "first")+ await accept_client_message("sess-1", "msg-a", "first")++ assert fake_redis.strings[key] == "reserved:successor"+++async def test_lapsed_claimant_still_records_its_accepted_send(+ fake_redis: _FakeRedis,+) -> None:+ """Nobody took the id over, so the send is marked accepted anyway."""+ key = pm_module._client_message_key("sess-1", "msg-a")+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ del fake_redis.strings[key] # the reservation expired++ await accept_client_message("sess-1", "msg-a", "first")++ assert await client_message_state("sess-1", "msg-a") == "accepted"+++async def test_accepting_a_claim_finishes_when_the_request_is_cancelled(+ fake_redis: _FakeRedis, monkeypatch: pytest.MonkeyPatch+) -> None:+ """The send already landed, so a cancelled request must still record it:+ a reservation left to lapse would let a retry send it again."""+ key = pm_module._client_message_key("sess-1", "msg-a")+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ accepting = asyncio.Event()+ finish = asyncio.Event()+ settle = fake_redis.eval++ async def slow_eval(*args: Any) -> int:+ accepting.set()+ await finish.wait()+ return await settle(*args)++ monkeypatch.setattr(fake_redis, "eval", slow_eval)+ request = asyncio.create_task(accept_client_message("sess-1", "msg-a", "first"))+ await accepting.wait()++ request.cancel()+ await asyncio.sleep(0)+ finish.set()++ with pytest.raises(asyncio.CancelledError):+ await request+ assert fake_redis.strings[key] == "accepted"+++async def test_accepting_a_claim_finishes_when_the_request_is_cancelled_twice(+ fake_redis: _FakeRedis, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A second cancellation while the accept is finishing must not cut it+ short either, or the reservation is left to lapse."""+ key = pm_module._client_message_key("sess-1", "msg-a")+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ accepting = asyncio.Event()+ finish = asyncio.Event()+ settle = fake_redis.eval++ async def slow_eval(*args: Any) -> int:+ accepting.set()+ await finish.wait()+ return await settle(*args)++ monkeypatch.setattr(fake_redis, "eval", slow_eval)+ request = asyncio.create_task(accept_client_message("sess-1", "msg-a", "first"))+ await accepting.wait()++ request.cancel()+ await asyncio.sleep(0)+ await asyncio.sleep(0)+ request.cancel()+ await asyncio.sleep(0)+ finish.set()++ with pytest.raises(asyncio.CancelledError):+ await request+ assert fake_redis.strings[key] == "accepted"+++async def test_copy_during_reservation_waits_for_acceptance(+ fake_redis: _FakeRedis, short_claim_wait: None+) -> None:+ """A copy is not told the send was taken until it has been."""+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ copy = asyncio.create_task(claim_client_message("sess-1", "msg-a", "copy"))+ await asyncio.sleep(0.02)+ assert not copy.done()++ await accept_client_message("sess-1", "msg-a", "first")++ assert await copy == "accepted"+++async def test_copy_during_reservation_takes_the_id_when_the_first_is_refused(+ fake_redis: _FakeRedis, short_claim_wait: None+) -> None:+ """The first copy was refused, so the send was never taken: the copy+ carries it instead of being answered as a duplicate."""+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ copy = asyncio.create_task(claim_client_message("sess-1", "msg-a", "copy"))+ await asyncio.sleep(0.02)++ await release_client_message("sess-1", "msg-a", "first")++ assert await copy == "claimed"+++async def test_copy_gives_up_when_the_reservation_outlasts_the_wait(+ fake_redis: _FakeRedis, short_claim_wait: None+) -> None:+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"++ assert await claim_client_message("sess-1", "msg-a", "copy") == "reserved"+ assert await client_message_state("sess-1", "msg-a") == "reserved"+++async def test_client_message_claims_fail_open_on_redis_error(+ monkeypatch: pytest.MonkeyPatch,+) -> None:+ """Dedup is best effort: a Redis blip must never block a send."""+ monkeypatch.setattr(+ pm_module, "get_redis_async", AsyncMock(side_effect=ConnectionError("down"))+ )+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"+ assert await client_message_state("sess-1", "msg-a") is None+ await accept_client_message("sess-1", "msg-a", "first")+ await release_client_message("sess-1", "msg-a", "first")diff --git a/autogpt_platform/backend/backend/data/redis_helpers.py b/autogpt_platform/backend/backend/data/redis_helpers.pyindex 71059eb275..c617400b61 100644--- a/autogpt_platform/backend/backend/data/redis_helpers.py+++ b/autogpt_platform/backend/backend/data/redis_helpers.py@@ -13,6 +13,8 @@ patterns we actually use into a single place:- :func:`hash_compare_and_set` — set a hash field only if its currentvalue matches an expected one. Genuinely needs Lua because thecondition depends on the current value (pipeline can't branch).+- :func:`string_compare_and_set` — the same for a plain string key, with+ delete as the "new value" option (ownership-checked release/handover).Everything sharable lives here. If a new Lua script is tempting inapplication code, add a helper here first — callers should not touch@@ -63,6 +65,26 @@ return 1# ARGV[3] list value# ARGV[4] max list length# ARGV[5] list TTL seconds+# Compare-and-set on a string key. Returns 1 if swapped, 0 otherwise.+#+# KEYS[1] key+# ARGV[1] expected current value+# ARGV[2] new value, or "" to delete the key+# ARGV[3] TTL in seconds for the new value+# ARGV[4] "1" to also swap when the key is missing+_STRING_CAS_LUA = """+local current = redis.call('GET', KEYS[1])+if current ~= ARGV[1] and not (current == false and ARGV[4] == '1') then+ return 0+end+if ARGV[2] == '' then+ redis.call('DEL', KEYS[1])+else+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])+end+return 1+"""+_GATED_CAPPED_RPUSH_LUA = """local current = redis.call('HGET', KEYS[1], ARGV[1])if current ~= ARGV[2] then@@ -433,3 +455,34 @@ async def hash_compare_and_set(redis.eval(_HASH_CAS_LUA, 1, key, field, expected, new, *(guard or ())),)return int(result) == 1+++async def string_compare_and_set(+ redis: AsyncRedisClient,+ key: str,+ *,+ expected: str,+ new: str | None,+ ttl_seconds: int = 0,+ or_missing: bool = False,+) -> bool:+ """Atomically replace *key* with *new* (``None`` deletes it) iff it still+ holds *expected*, or is missing and *or_missing* is set.++ Returns ``True`` if the swap happened. Use it when a value names its+ owner, so a holder whose key expired and was taken over cannot overwrite+ or delete its successor's value.+ """+ result = await cast(+ "Any",+ redis.eval(+ _STRING_CAS_LUA,+ 1,+ key,+ expected,+ new or "",+ ttl_seconds,+ "1" if or_missing else "0",+ ),+ )+ return int(result) == 1diff --git a/autogpt_platform/backend/backend/data/redis_helpers_test.py b/autogpt_platform/backend/backend/data/redis_helpers_test.pyindex 9ce31d7faf..edaff125ac 100644--- a/autogpt_platform/backend/backend/data/redis_helpers_test.py+++ b/autogpt_platform/backend/backend/data/redis_helpers_test.py@@ -17,6 +17,7 @@ from backend.data.redis_helpers import (hash_compare_and_set,incr_with_ttl,incr_with_ttl_sync,+ string_compare_and_set,)# ── Fake Redis + pipeline ──────────────────────────────────────────────@@ -87,6 +88,19 @@ class _Fake:await self.expire(list_key, int(ttl_seconds))return await self.llen(list_key)+ if "'GET'" in script:+ # ``string_compare_and_set`` shape.+ key, expected, new, ttl_seconds, or_missing = args[:5]+ current = self.strings.get(key)+ if current != expected and not (current is None and or_missing == "1"):+ return 0+ if new:+ self.strings[key] = new+ self.ttls[key] = int(ttl_seconds)+ else:+ self.strings.pop(key, None)+ return 1+# ``hash_compare_and_set`` shape (numkeys == 1).key, field, expected, new = args[0], args[1], args[2], args[3]h = self.hashes.setdefault(key, {})@@ -298,6 +312,57 @@ async def test_hash_cas_no_swap_when_expected_differs() -> None:assert r.hashes["meta"]["status"] == "completed"+# ── string_compare_and_set ─────────────────────────────────────────────+++async def test_string_cas_swaps_and_sets_the_ttl_when_expected_matches() -> None:+ r = _Fake()+ r.strings["claim"] = "reserved:a"+ swapped = await string_compare_and_set(+ r, "claim", expected="reserved:a", new="accepted", ttl_seconds=60 # type: ignore[arg-type]+ )+ assert swapped is True+ assert r.strings["claim"] == "accepted"+ assert r.ttls["claim"] == 60+++async def test_string_cas_deletes_when_new_is_none() -> None:+ r = _Fake()+ r.strings["claim"] = "reserved:a"+ swapped = await string_compare_and_set(+ r, "claim", expected="reserved:a", new=None # type: ignore[arg-type]+ )+ assert swapped is True+ assert "claim" not in r.strings+++async def test_string_cas_leaves_another_holders_value_alone() -> None:+ r = _Fake()+ r.strings["claim"] = "reserved:b"+ assert not await string_compare_and_set(+ r, "claim", expected="reserved:a", new=None # type: ignore[arg-type]+ )+ assert not await string_compare_and_set(+ r, "claim", expected="reserved:a", new="accepted", or_missing=True # type: ignore[arg-type]+ )+ assert r.strings["claim"] == "reserved:b"+++async def test_string_cas_or_missing_sets_a_missing_key() -> None:+ r = _Fake()+ assert not await string_compare_and_set(+ r, "claim", expected="reserved:a", new="accepted" # type: ignore[arg-type]+ )+ assert await string_compare_and_set(+ r, "claim", expected="reserved:a", new="accepted", or_missing=True # type: ignore[arg-type]+ )+ assert r.strings["claim"] == "accepted"++# ── claim_batch_dispatch_atomic ───────────────────────────────────────diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.tsindex 7a4df0def5..a8081d9d84 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts@@ -60,9 +60,36 @@ describe("queueFollowUpMessage", () => {message: "hello",context: null,file_ids: null,+ message_id: expect.stringMatching(+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,+ ),});});+ it("sends a fresh idempotency key for every queued message", async () => {+ const fetchMock = vi.mocked(global.fetch);+ for (let i = 0; i < 2; i++) {+ fetchMock.mockResolvedValueOnce(+ new Response(+ JSON.stringify({+ buffer_length: i + 1,+ max_buffer_length: 10,+ turn_in_flight: true,+ }),+ { status: 200, headers: { "Content-Type": "application/json" } },+ ),+ );+ }++ await queueFollowUpMessage("sess-1", "same text");+ await queueFollowUpMessage("sess-1", "same text");++ const ids = fetchMock.mock.calls.map(+ ([, init]) => JSON.parse(init?.body as string).message_id,+ );+ expect(ids[0]).not.toEqual(ids[1]);+ });+it("treats a 200 inactive response as not active", async () => {const fetchMock = vi.mocked(global.fetch);fetchMock.mockResolvedValueOnce(diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.tsindex 0267369902..59867ef371 100644--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts@@ -1,4 +1,5 @@import { environment } from "@/services/environment";+import { v4 as uuidv4 } from "uuid";import { getCopilotAuthHeaders } from "../helpers";@@ -34,6 +35,10 @@ export async function queueFollowUpMessage(message,context: null,file_ids: null,+ // Idempotency key: a retransmit of this request is queued only once.+ // Options force uuid's getRandomValues path, which (unlike+ // crypto.randomUUID) exists on plain-HTTP LAN origins.+ message_id: uuidv4({}),}),});diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.jsonindex bfccd3c7b0..4fd7d69f7c 100644--- a/autogpt_platform/frontend/src/app/api/openapi.json+++ b/autogpt_platform/frontend/src/app/api/openapi.json@@ -28548,6 +28548,14 @@{ "type": "null" }],"title": "Folder Ids"+ },+ "message_id": {+ "anyOf": [+ { "type": "string", "maxLength": 64 },+ { "type": "null" }+ ],+ "title": "Message Id",+ "description": "Optional per-send UUID, scoped like ``StreamChatRequest.message_id``. A retransmit of a follow-up already queued is answered as accepted without queueing it again."}},"type": "object",@@ -31023,7 +31031,7 @@{ "type": "null" }],"title": "Message Id",- "description": "Optional per-click UUID generated by the frontend. The server scopes it to the authenticated user and session before using the result as ``ChatMessage.id``. Frontend / network / RMQ-redelivery retransmits of the same logical send reuse the key, so the Postgres unique-constraint on the resulting PK is the atomic dedup primitive. A duplicate INSERT returns a subscribe-only response without creating a parallel turn. Distinct user clicks (even with identical text) MUST send different ids — the frontend's per-click ``crypto.randomUUID()`` guarantees that."+ "description": "Optional per-click UUID generated by the frontend. The server scopes it to the authenticated user and session before using the result as ``ChatMessage.id``. Frontend / network / RMQ-redelivery retransmits of the same logical send reuse the key, and the server claims it once: a retransmit neither starts a parallel turn nor, while a turn is running, queues the message again, and gets a subscribe-only response. Distinct user clicks (even with identical text) MUST send different ids — the frontend's per-click ``crypto.randomUUID()`` guarantees that."},"expert_kickoff": {"type": "boolean",