Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

[SECRT-2695] fix(platform): stop a retransmitted chat send from queueing and running again (#14939)

Bentlyd90300d5722c
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes.py b/autogpt_platform/backend/backend/api/features/chat/routes.py
index 51df0a5cc6..209d87a987 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes.py
@@ -67,13 +67,19 @@ from backend.copilot.offers import (
from backend.copilot.pending_message_helpers import (
QueuePendingMessageResponse,
StreamRegistryUnavailable,
+ already_accepted_response,
+ client_message_in_progress_error,
is_turn_in_flight,
queue_pending_for_http,
resolve_attachments_for_http,
)
from backend.copilot.pending_messages import (
+ accept_client_message,
+ claim_client_message,
clear_pending_messages_unsafe,
+ client_message_state,
peek_pending_messages,
+ release_client_message,
)
from backend.copilot.provider_failure import ProviderFailure, ProviderFailureKind
from backend.copilot.provider_tiers import (
@@ -317,9 +323,9 @@ class StreamChatRequest(BaseModel):
"scopes it to the authenticated user and session before using "
"the result as ``ChatMessage.id``. Frontend / network / "
"RMQ-redelivery retransmits of the same logical send reuse the "
- "key, so the Postgres unique-constraint on the resulting PK is "
- "the atomic dedup primitive. A duplicate INSERT returns a "
- "subscribe-only response without creating a parallel turn. "
+ "key, and the server claims it once: a retransmit neither starts "
+ "a parallel turn nor, while a turn is running, queues the "
+ "message again, and gets a subscribe-only response. "
"Distinct user clicks (even with identical text) MUST send "
"different ids — the frontend's per-click ``crypto.randomUUID()`` "
"guarantees that."
@@ -346,6 +352,15 @@ class QueuePendingMessageRequest(BaseModel):
context: dict[str, str] | None = None
file_ids: list[str] | None = Field(default=None, max_length=20)
folder_ids: list[str] | None = Field(default=None, max_length=5)
+ message_id: str | None = Field(
+ default=None,
+ max_length=64,
+ description=(
+ "Optional per-send UUID, scoped like ``StreamChatRequest."
+ "message_id``. A retransmit of a follow-up already queued is "
+ "answered as accepted without queueing it again."
+ ),
+ )
class PeekPendingMessagesResponse(BaseModel):
@@ -1839,6 +1854,9 @@ async def stream_chat_post(
session_id,
)
+ # The kickoff owns a server-derived id and its own replay handling.
+ client_message_id = None if request.expert_kickoff else message_id
+
# Session-anchored tenancy: the ChatSession row is the authoritative
# org/team for every turn in it — a user whose active header org
# differs still charges/attributes turns to the session's org.
@@ -1878,6 +1896,7 @@ async def stream_chat_post(
file_ids=request.file_ids,
folder_ids=request.folder_ids,
expert_id=session.expert_id,
+ client_message_id=client_message_id,
)
return _empty_ui_message_stream_response(follow_up_queued=True)
except HTTPException as exc:
@@ -1977,29 +1996,55 @@ async def stream_chat_post(
# near the start) — that path returns early. Any request that
# reaches this point is starting a fresh turn, so we always mint a
# ``turn_id`` unless ``append_and_save_message`` reports a duplicate.
+ #
+ # The client key is claimed before the turn can go in flight. From then on
+ # a retransmit takes the queue branch above, where the PK cannot catch it:
+ # the row may not be written yet, and a queued copy never gets this id.
+ # A copy that finds the key reserved waits for this request's outcome, so
+ # it is never told a send was taken that is then refused.
+ claim_owner = uuid4().hex
+ claim = (
+ await claim_client_message(session_id, client_message_id, claim_owner)
+ if client_message_id is not None
+ else None
+ )
+ if claim == "reserved":
+ raise client_message_in_progress_error()
+
+ async def release_client_message_claim() -> None:
+ if claim == "claimed" and client_message_id is not None:
+ await release_client_message(session_id, client_message_id, claim_owner)
+
+ async def accept_client_message_claim() -> None:
+ if claim == "claimed" and client_message_id is not None:
+ await accept_client_message(session_id, client_message_id, claim_owner)
+
try:
- turn_id = await schedule_chat_turn(
- session_id=session_id,
- user_id=user_id,
- message=message,
- message_id=message_id,
- message_metadata=message_metadata,
- message_already_persisted=resume_persisted_kickoff,
- is_user_message=request.is_user_message,
- expert_id=session.expert_id,
- session_origin=session.metadata.origin,
- session_source_platform=session.metadata.source_platform,
- context=request.context,
- voice=request.voice,
- file_ids=sanitized_file_ids,
- organization_id=turn_org_id,
- team_id=turn_team_id,
- model=request.model,
- llm_auth_provider=session.metadata.llm_auth_provider,
- llm_credential_id=session.metadata.llm_credential_id,
- permissions=builder_permissions,
- request_arrival_at=request_arrival_at,
- )
+ if claim == "accepted":
+ turn_id = None
+ else:
+ turn_id = await schedule_chat_turn(
+ session_id=session_id,
+ user_id=user_id,
+ message=message,
+ message_id=message_id,
+ message_metadata=message_metadata,
+ message_already_persisted=resume_persisted_kickoff,
+ is_user_message=request.is_user_message,
+ expert_id=session.expert_id,
+ session_origin=session.metadata.origin,
+ session_source_platform=session.metadata.source_platform,
+ context=request.context,
+ voice=request.voice,
+ file_ids=sanitized_file_ids,
+ organization_id=turn_org_id,
+ team_id=turn_team_id,
+ model=request.model,
+ llm_auth_provider=session.metadata.llm_auth_provider,
+ llm_credential_id=session.metadata.llm_credential_id,
+ permissions=builder_permissions,
+ request_arrival_at=request_arrival_at,
+ )
except ConcurrentTurnLimitError as exc:
if resume_persisted_kickoff:
raise HTTPException(status_code=429, detail=str(exc)) from exc
@@ -2031,15 +2076,26 @@ async def stream_chat_post(
request_arrival_at=request_arrival_at,
)
except turn_queue.InflightCapExceeded:
+ await release_client_message_claim()
raise HTTPException(
status_code=429,
detail=inflight_turn_limit_message(inflight_cap),
)
+ except BaseException:
+ # Raised inside the handler above, so the outer BaseException
+ # clause never sees it.
+ await release_client_message_claim()
+ raise
+ await accept_client_message_claim()
logger.info(
f"[STREAM] Queued turn for session={session_id} "
f"(running cap reached; inflight cap={inflight_cap})"
)
return _empty_ui_message_stream_response()
+ except BaseException:
+ await release_client_message_claim()
+ raise
+ await accept_client_message_claim()
if turn_id is None:
logger.info(
@@ -2244,6 +2300,19 @@ async def queue_pending_message(
session = await _validate_and_get_writable_session(session_id, user_id)
if session.metadata.llm_auth_provider == "codex":
await enforce_codex_access_http(user_id)
+ client_message_id = (
+ scoped_client_message_id(user_id, session_id, request.message_id)
+ if request.message_id
+ else None
+ )
+ # Before the in-flight gate: a retransmit landing after the turn drained
+ # its original must not 409, or the client re-sends it via POST /stream.
+ if client_message_id is not None:
+ state = await client_message_state(session_id, client_message_id)
+ if state == "accepted":
+ return await already_accepted_response(session_id)
+ if state == "reserved":
+ raise client_message_in_progress_error()
try:
turn_in_flight = await is_turn_in_flight(session_id)
except StreamRegistryUnavailable as exc:
@@ -2265,6 +2334,7 @@ async def queue_pending_message(
file_ids=request.file_ids,
folder_ids=request.folder_ids,
expert_id=session.expert_id,
+ client_message_id=client_message_id,
)
diff --git a/autogpt_platform/backend/backend/api/features/chat/routes_test.py b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
index f9f3ed6a6f..4ab3d61c54 100644
--- a/autogpt_platform/backend/backend/api/features/chat/routes_test.py
+++ b/autogpt_platform/backend/backend/api/features/chat/routes_test.py
@@ -8,6 +8,7 @@ from unittest.mock import AsyncMock, MagicMock
import fastapi
import fastapi.testclient
+import httpx
import pytest
import pytest_mock
from pydantic import SecretStr
@@ -15,6 +16,7 @@ from sentry_sdk.integrations.asgi import SentryAsgiMiddleware
from backend.api.features.chat import routes as chat_routes
from backend.api.features.chat.routes import _strip_injected_context
+from backend.copilot import pending_messages as pending_messages_module
from backend.copilot import transports as chat_transports
from backend.copilot.config import CopilotLlmAuthProvider
from backend.copilot.model import ChatSession
@@ -347,11 +349,13 @@ def _mock_stream_internals(
new_callable=AsyncMock,
return_value=None,
)
+ claims = _mock_client_message_claims(mocker)
return types.SimpleNamespace(
enqueue=mock_schedule,
paywall=mock_paywall,
session=mock_session,
owns_active_expert=mock_owns_active_expert,
+ claims=claims,
)
@@ -741,6 +745,281 @@ def test_stream_chat_scopes_client_message_id_to_owner_and_session(
)
+# ─── Client message id idempotency (SECRT-2695) ───────────────────────
+
+
+class _ClaimRedis:
+ """The client-message claim's SET NX / GET / compare-and-set, plus the
+ LLEN a duplicate's response reads the buffer length with."""
+
+ def __init__(self) -> None:
+ self.values: dict[str, str] = {}
+
+ @property
+ def keys(self) -> set[str]:
+ return set(self.values)
+
+ async def set(
+ self, key: str, value: str, nx: bool = False, ex: int | None = None
+ ) -> bool | None:
+ if nx and key in self.values:
+ return None
+ self.values[key] = value
+ return True
+
+ async def get(self, key: str) -> str | None:
+ return self.values.get(key)
+
+ async def eval(
+ self,
+ script: str,
+ numkeys: int,
+ key: str,
+ expected: str,
+ new: str,
+ ttl_seconds: int,
+ or_missing: str,
+ ) -> int:
+ """``string_compare_and_set``, which settles a claim."""
+ current = self.values.get(key)
+ if current != expected and not (current is None and or_missing == "1"):
+ return 0
+ if new:
+ self.values[key] = new
+ else:
+ self.values.pop(key, None)
+ return 1
+
+ async def llen(self, key: str) -> int:
+ return 0
+
+
+def _mock_client_message_claims(mocker: pytest_mock.MockerFixture) -> _ClaimRedis:
+ redis = _ClaimRedis()
+ mocker.patch(
+ "backend.copilot.pending_messages.get_redis_async",
+ new_callable=AsyncMock,
+ return_value=redis,
+ )
+ return redis
+
+
+def _mock_pending_push(mocker: pytest_mock.MockerFixture) -> AsyncMock:
+ """Run the real ``queue_pending_for_http`` down to the Redis push."""
+ mocker.patch(
+ "backend.copilot.pending_message_helpers.get_redis_async",
+ new_callable=AsyncMock,
+ return_value=mocker.MagicMock(),
+ )
+ mocker.patch(
+ "backend.copilot.pending_message_helpers.incr_with_ttl",
+ new_callable=AsyncMock,
+ return_value=1,
+ )
+ return mocker.patch(
+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",
+ new_callable=AsyncMock,
+ return_value=1,
+ )
+
+
+def test_stream_chat_retransmits_of_a_started_turn_queue_nothing(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """The same send arriving three times starts one turn. The copies that
+ land once the turn is running used to be buffered as follow-ups, each
+ one becoming another user message and another turn."""
+ mocks = _mock_stream_internals(mocker)
+ push = _mock_pending_push(mocker)
+ in_flight = mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ new_callable=AsyncMock,
+ return_value=False,
+ )
+ body = {"message": "hello", "message_id": "client-click-id"}
+
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+ in_flight.return_value = True
+ for _ in range(2):
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+
+ mocks.enqueue.assert_awaited_once()
+ push.assert_not_awaited()
+
+
+def test_stream_chat_retransmits_of_a_queued_send_are_buffered_once(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """A stale tab posts to /stream while another tab's turn runs, so the
+ send is queued. Its copies must neither queue again nor, once the turn
+ has drained it, start a turn of their own."""
+ mocks = _mock_stream_internals(mocker)
+ push = _mock_pending_push(mocker)
+ in_flight = mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ new_callable=AsyncMock,
+ return_value=True,
+ )
+ body = {"message": "hello", "message_id": "client-click-id"}
+
+ for _ in range(3):
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+ in_flight.return_value = False
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+
+ push.assert_awaited_once()
+ mocks.enqueue.assert_not_awaited()
+
+
+def test_stream_chat_distinct_message_ids_are_each_queued(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """Two clicks with the same text are two messages."""
+ _mock_stream_internals(mocker)
+ push = _mock_pending_push(mocker)
+ mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ new_callable=AsyncMock,
+ return_value=True,
+ )
+
+ for message_id in ("click-1", "click-2"):
+ response = client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello", "message_id": message_id},
+ )
+ assert response.status_code == 200
+
+ assert push.await_count == 2
+
+
+def test_stream_chat_refused_send_can_be_retried_with_the_same_id(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """A send the server turned away was never accepted, so its id must not
+ stay claimed: the retry would be dropped as a duplicate of nothing."""
+ mocks = _mock_stream_internals(mocker)
+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")
+ mocker.patch.object(
+ chat_routes.turn_queue,
+ "try_enqueue_turn",
+ new_callable=AsyncMock,
+ side_effect=chat_routes.turn_queue.InflightCapExceeded(),
+ )
+ body = {"message": "hello", "message_id": "client-click-id"}
+
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 429
+ assert mocks.claims.keys == set()
+
+ mocks.enqueue.side_effect = None
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+ assert mocks.enqueue.await_count == 2
+
+
+def test_stream_chat_failed_queue_fallback_can_be_retried_with_the_same_id(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """The running cap sends the turn to the queue. If that fails too, the
+ send was never accepted and its id must be free for the retry."""
+ mocks = _mock_stream_internals(mocker)
+ mocks.enqueue.side_effect = chat_routes.ConcurrentTurnLimitError("busy")
+ mocker.patch.object(
+ chat_routes.turn_queue,
+ "try_enqueue_turn",
+ new_callable=AsyncMock,
+ side_effect=RuntimeError("database unavailable"),
+ )
+ body = {"message": "hello", "message_id": "client-click-id"}
+ safe_client = fastapi.testclient.TestClient(app, raise_server_exceptions=False)
+
+ response = safe_client.post("/sessions/sess-1/stream", json=body)
+ assert response.status_code == 500
+ assert mocks.claims.keys == set()
+
+ mocks.enqueue.side_effect = None
+ assert client.post("/sessions/sess-1/stream", json=body).status_code == 200
+ assert mocks.enqueue.await_count == 2
+
+
+async def test_stream_chat_copy_carries_a_send_the_first_copy_had_refused(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """A copy lands while the first copy is being scheduled, and the first is
+ then refused at the in-flight cap. The send was never taken, so the copy
+ must schedule it, not be answered as a duplicate of nothing."""
+ mocks = _mock_stream_internals(mocker)
+ first_scheduling = asyncio.Event()
+ let_first_finish = asyncio.Event()
+ scheduled: list[str] = []
+
+ async def schedule(**kwargs: object) -> str:
+ scheduled.append("call")
+ if len(scheduled) == 1:
+ first_scheduling.set()
+ await let_first_finish.wait()
+ raise chat_routes.ConcurrentTurnLimitError("busy")
+ return "turn-2"
+
+ mocks.enqueue.side_effect = schedule
+ mocker.patch.object(
+ chat_routes.turn_queue,
+ "try_enqueue_turn",
+ new_callable=AsyncMock,
+ side_effect=chat_routes.turn_queue.InflightCapExceeded(),
+ )
+ body = {"message": "hello", "message_id": "client-click-id"}
+
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=app), base_url="http://test"
+ ) as async_client:
+ first = asyncio.create_task(
+ async_client.post("/sessions/sess-1/stream", json=body)
+ )
+ await first_scheduling.wait()
+ copy = asyncio.create_task(
+ async_client.post("/sessions/sess-1/stream", json=body)
+ )
+ await asyncio.sleep(0.2)
+ let_first_finish.set()
+ first_response = await first
+ copy_response = await copy
+
+ assert first_response.status_code == 429
+ assert copy_response.status_code == 200
+ assert len(scheduled) == 2
+ assert list(mocks.claims.values.values()) == ["accepted"]
+
+
+def _reserve_client_message(claims: _ClaimRedis, user_id: str) -> None:
+ """Hold the id the way a first copy still being scheduled does."""
+ scoped_id = chat_routes.scoped_client_message_id(
+ user_id, "sess-1", "client-click-id"
+ )
+ key = pending_messages_module._client_message_key("sess-1", scoped_id)
+ claims.values[key] = "reserved:first-copy"
+
+
+def test_stream_chat_copy_is_told_to_retry_while_the_first_is_unsettled(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+) -> None:
+ """The first copy is still being scheduled and may yet be refused, so
+ this one is told neither that the send was taken nor that it failed."""
+ mocks = _mock_stream_internals(mocker)
+ mocker.patch.object(pending_messages_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)
+ _reserve_client_message(mocks.claims, test_user_id)
+
+ response = client.post(
+ "/sessions/sess-1/stream",
+ json={"message": "hello", "message_id": "client-click-id"},
+ )
+
+ assert response.status_code == 503
+ mocks.enqueue.assert_not_awaited()
+ assert list(mocks.claims.values.values()) == ["reserved:first-copy"]
+
+
# ─── UUID format filtering ─────────────────────────────────────────────
@@ -2379,6 +2658,60 @@ def test_queue_pending_message_returns_200_when_turn_in_flight(
assert "turn_in_flight" in data
+def test_queue_pending_message_same_message_id_is_buffered_once(
+ mocker: pytest_mock.MockerFixture,
+) -> None:
+ """A retransmitted follow-up is queued once, and a copy that lands after
+ the turn ended is answered as accepted rather than 409 (which would send
+ the client to POST /stream with the message all over again)."""
+ _mock_stream_queue_internals(mocker)
+ _mock_client_message_claims(mocker)
+ push = mocker.patch(
+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",
+ new_callable=AsyncMock,
+ return_value=1,
+ )
+ body = {"message": "follow-up", "message_id": "client-click-id"}
+
+ for _ in range(2):
+ response = client.post("/sessions/sess-1/messages/pending", json=body)
+ assert response.status_code == 200
+ assert response.json()["turn_in_flight"] is True
+ mocker.patch(
+ "backend.api.features.chat.routes.is_turn_in_flight",
+ new_callable=AsyncMock,
+ return_value=False,
+ )
+ response = client.post("/sessions/sess-1/messages/pending", json=body)
+
+ assert response.status_code == 200
+ assert response.json()["turn_in_flight"] is True
+ push.assert_awaited_once()
+
+
+def test_queue_pending_message_copy_is_told_to_retry_while_the_first_is_unsettled(
+ mocker: pytest_mock.MockerFixture,
+ test_user_id: str,
+) -> None:
+ _mock_stream_queue_internals(mocker)
+ claims = _mock_client_message_claims(mocker)
+ mocker.patch.object(pending_messages_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)
+ _reserve_client_message(claims, test_user_id)
+ push = mocker.patch(
+ "backend.copilot.pending_message_helpers.push_pending_message_if_session_running",
+ new_callable=AsyncMock,
+ return_value=1,
+ )
+
+ response = client.post(
+ "/sessions/sess-1/messages/pending",
+ json={"message": "follow-up", "message_id": "client-click-id"},
+ )
+
+ assert response.status_code == 503
+ push.assert_not_awaited()
+
+
def test_queue_pending_message_session_not_found_returns_404(
mocker: pytest_mock.MockerFixture,
) -> None:
diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
index 52f9f6faa4..ce540b56a5 100644
--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers.py
@@ -7,7 +7,9 @@ Also provides the call-rate-limit check for the queue endpoint so
routes.py stays free of Redis/Lua details.
"""
+import asyncio
import logging
+import uuid
from typing import TYPE_CHECKING, Any, Callable
from fastapi import HTTPException
@@ -24,10 +26,15 @@ from backend.copilot.pending_messages import (
MAX_PENDING_MESSAGES,
PendingMessage,
PendingMessageContext,
+ accept_client_message,
+ claim_client_message,
drain_pending_messages,
+ finish_despite_cancellation,
format_pending_as_user_message,
+ peek_pending_count,
push_pending_message,
push_pending_message_if_session_running,
+ release_client_message,
)
from backend.copilot.response_model import (
StreamPendingDrained,
@@ -211,6 +218,7 @@ async def queue_pending_for_http(
file_ids: list[str] | None,
folder_ids: list[str] | None,
expert_id: str | None,
+ client_message_id: str | None = None,
) -> QueuePendingMessageResponse:
"""HTTP-facing wrapper around :func:`queue_user_message`.
@@ -226,6 +234,13 @@ async def queue_pending_for_http(
as a field: a pending message is rendered from ``content`` when the turn
drains it, and entries written by older workers are still in Redis.
+ ``client_message_id`` is the send's scoped idempotency key. A send
+ whose key was already accepted is answered without pushing again (see
+ :func:`already_accepted_response`), and a push that does not land gives
+ the key back so a genuine retry can. A copy that arrives while another
+ is still being pushed waits for its outcome, and gets a 503 if that
+ outlasts the wait.
+
Raises :class:`HTTPException` with status 429 if the rate cap is hit or
400 if an expert session attaches a file outside its scope; otherwise
returns the ``QueuePendingMessageResponse`` the handler can serialise 1:1.
@@ -256,18 +271,56 @@ async def queue_pending_for_http(
# the FE's is_turn_in_flight check and our gate), which both this
# endpoint and the POST /stream queue-fall-through can hit. Pushing
# first lets the gate own the no-op short-circuit.
- response = await queue_user_message(
- session_id=session_id,
- message=message,
- context=queue_context,
- file_ids=sanitized_file_ids,
- require_turn_in_flight=True,
+ claim_owner = uuid.uuid4().hex
+ if client_message_id is not None:
+ claim = await claim_client_message(session_id, client_message_id, claim_owner)
+ if claim == "accepted":
+ logger.info(
+ "pending_messages: skipped retransmit of an accepted message "
+ "for session=%s",
+ session_id,
+ )
+ return await already_accepted_response(session_id)
+ if claim == "reserved":
+ raise client_message_in_progress_error()
+
+ push = asyncio.ensure_future(
+ queue_user_message(
+ session_id=session_id,
+ message=message,
+ context=queue_context,
+ file_ids=sanitized_file_ids,
+ require_turn_in_flight=True,
+ )
)
+ try:
+ response = await asyncio.shield(push)
+ except asyncio.CancelledError:
+ # The append may already have landed (the push still awaits its
+ # SPUBLISH), so settle the claim by what the push did, not by the
+ # cancellation: releasing it would let a retry queue a second copy.
+ if client_message_id is not None:
+ await finish_despite_cancellation(
+ asyncio.ensure_future(
+ _settle_claim_of_cancelled_push(
+ push, session_id, client_message_id, claim_owner
+ )
+ )
+ )
+ raise
+ except BaseException:
+ if client_message_id is not None:
+ await release_client_message(session_id, client_message_id, claim_owner)
+ raise
if not response.turn_in_flight:
+ if client_message_id is not None:
+ await release_client_message(session_id, client_message_id, claim_owner)
raise HTTPException(
status_code=409,
detail="Session has no active turn. Start a new turn with POST /stream.",
)
+ if client_message_id is not None:
+ await accept_client_message(session_id, client_message_id, claim_owner)
# Push landed — now charge the rate counter. If this tick crosses the
# limit we still keep the queued message (next drain will pick it up)
@@ -286,6 +339,60 @@ async def queue_pending_for_http(
return response
+async def _settle_claim_of_cancelled_push(
+ push: "asyncio.Future[QueuePendingMessageResponse]",
+ session_id: str,
+ client_message_id: str,
+ owner: str,
+) -> None:
+ try:
+ response = await push
+ except Exception:
+ await release_client_message(session_id, client_message_id, owner)
+ return
+ if response.turn_in_flight:
+ await accept_client_message(session_id, client_message_id, owner)
+ else:
+ await release_client_message(session_id, client_message_id, owner)
+
+
+def client_message_in_progress_error() -> HTTPException:
+ """For a copy that arrived while the first copy was still being scheduled
+ and outlasted the wait for its outcome. It is neither accepted nor
+ refused yet, so the client is told to retry, as for a degraded registry.
+ """
+ return HTTPException(
+ status_code=503,
+ detail="Your message is still being sent, retry shortly",
+ headers={"Retry-After": "5"},
+ )
+
+
+async def already_accepted_response(session_id: str) -> QueuePendingMessageResponse:
+ """Answer a retransmit of a send the server already accepted.
+
+ ``turn_in_flight`` is ``True`` whatever the turn is doing now: it tells the
+ client its message was taken, and ``False`` would make it fall back to
+ ``POST /stream`` and send the message all over again.
+ """
+ try:
+ buffer_length = await peek_pending_count(session_id)
+ except Exception as e:
+ # The count is informational. A Redis blip must not turn an accepted
+ # send into an error, which the client would answer by resending it.
+ logger.warning(
+ "pending_messages: buffer length lookup failed for session=%s: %s",
+ session_id,
+ e,
+ )
+ buffer_length = 0
+ return QueuePendingMessageResponse(
+ buffer_length=buffer_length,
+ max_buffer_length=MAX_PENDING_MESSAGES,
+ turn_in_flight=True,
+ )
+
+
async def check_pending_call_rate(user_id: str) -> int:
"""Increment and return the per-user push counter for the current window.
diff --git a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
index ab9e0a7b30..95a9afc8f3 100644
--- a/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
+++ b/autogpt_platform/backend/backend/copilot/pending_message_helpers_test.py
@@ -1,8 +1,9 @@
"""Unit tests for pending_message_helpers."""
+import asyncio
from types import SimpleNamespace
from typing import Any
-from unittest.mock import AsyncMock, MagicMock
+from unittest.mock import ANY, AsyncMock, MagicMock
import pytest
from fastapi import HTTPException
@@ -284,6 +285,241 @@ async def test_queue_pending_429_after_push_when_limit_exceeded(
queue_mock.assert_awaited_once()
+# ── queue_pending_for_http: client message id (SECRT-2695) ──────────
+
+
+def _mock_queue_claims(
+ monkeypatch: pytest.MonkeyPatch, *, claim_result: str
+) -> tuple[AsyncMock, AsyncMock, AsyncMock]:
+ claim = AsyncMock(return_value=claim_result)
+ accept = AsyncMock()
+ release = AsyncMock()
+ monkeypatch.setattr(helpers_module, "claim_client_message", claim)
+ monkeypatch.setattr(helpers_module, "accept_client_message", accept)
+ monkeypatch.setattr(helpers_module, "release_client_message", release)
+ monkeypatch.setattr(helpers_module, "peek_pending_count", AsyncMock(return_value=1))
+ return claim, accept, release
+
+
+async def _queue_with_client_id() -> QueuePendingMessageResponse:
+ return await queue_pending_for_http(
+ session_id="sess-1",
+ user_id="user-1",
+ message="hi",
+ context=None,
+ file_ids=None,
+ folder_ids=None,
+ expert_id=None,
+ client_message_id="scoped-msg-1",
+ )
+
+
+async def test_queue_pending_skips_retransmit_of_an_accepted_message(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A retransmit is answered as accepted without a second push or a
+ rate tick, and says the turn is in flight so the client does not fall
+ back to POST /stream."""
+ _mock_queue_claims(monkeypatch, claim_result="accepted")
+ queue_mock = AsyncMock()
+ monkeypatch.setattr(helpers_module, "queue_user_message", queue_mock)
+ rate_mock = AsyncMock(return_value=1)
+ monkeypatch.setattr(helpers_module, "check_pending_call_rate", rate_mock)
+
+ result = await _queue_with_client_id()
+
+ assert result.turn_in_flight is True
+ assert result.buffer_length == 1
+ queue_mock.assert_not_awaited()
+ rate_mock.assert_not_awaited()
+
+
+async def test_queue_pending_keeps_the_claim_once_the_push_lands(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ claim, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")
+ monkeypatch.setattr(
+ helpers_module,
+ "queue_user_message",
+ AsyncMock(
+ return_value=QueuePendingMessageResponse(
+ buffer_length=1,
+ max_buffer_length=MAX_PENDING_MESSAGES,
+ turn_in_flight=True,
+ )
+ ),
+ )
+ monkeypatch.setattr(
+ helpers_module, "check_pending_call_rate", AsyncMock(return_value=1)
+ )
+
+ await _queue_with_client_id()
+
+ claim.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ release.assert_not_awaited()
+
+
+async def test_queue_pending_releases_the_claim_when_the_push_is_refused(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The turn ended before the push: the route falls through to start a
+ turn with the same id, which needs the claim back."""
+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")
+ monkeypatch.setattr(
+ helpers_module,
+ "queue_user_message",
+ AsyncMock(
+ return_value=QueuePendingMessageResponse(
+ buffer_length=0,
+ max_buffer_length=MAX_PENDING_MESSAGES,
+ turn_in_flight=False,
+ )
+ ),
+ )
+
+ with pytest.raises(HTTPException) as exc_info:
+ await _queue_with_client_id()
+
+ assert exc_info.value.status_code == 409
+ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ accept.assert_not_awaited()
+
+
+async def test_queue_pending_releases_the_claim_when_the_push_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")
+ monkeypatch.setattr(
+ helpers_module,
+ "queue_user_message",
+ AsyncMock(side_effect=ConnectionError("down")),
+ )
+
+ with pytest.raises(ConnectionError):
+ await _queue_with_client_id()
+
+ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ accept.assert_not_awaited()
+
+
[email protected]("landed", [True, False])
+async def test_queue_pending_settles_the_claim_by_the_push_when_cancelled(
+ monkeypatch: pytest.MonkeyPatch, landed: bool
+) -> None:
+ """The request is cancelled after the append, while the push still awaits
+ its SPUBLISH: the push finishes and the claim follows its real outcome,
+ so a retry of a message that is already queued is not pushed again."""
+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")
+ appended = asyncio.Event()
+ published = asyncio.Event()
+
+ async def push(**_: Any) -> QueuePendingMessageResponse:
+ appended.set()
+ await published.wait()
+ return QueuePendingMessageResponse(
+ buffer_length=1 if landed else 0,
+ max_buffer_length=MAX_PENDING_MESSAGES,
+ turn_in_flight=landed,
+ )
+
+ monkeypatch.setattr(helpers_module, "queue_user_message", push)
+ request = asyncio.create_task(_queue_with_client_id())
+ await appended.wait()
+ request.cancel()
+ await asyncio.sleep(0)
+ published.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await request
+
+ if landed:
+ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ release.assert_not_awaited()
+ else:
+ release.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ accept.assert_not_awaited()
+
+
+async def test_queue_pending_settles_the_claim_when_cancelled_again_while_settling(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """A second cancellation while the claim is being settled must not cut the
+ push or the settle short, or the reservation is left to lapse."""
+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="claimed")
+ appended = asyncio.Event()
+ published = asyncio.Event()
+
+ async def push(**_: Any) -> QueuePendingMessageResponse:
+ appended.set()
+ await published.wait()
+ return QueuePendingMessageResponse(
+ buffer_length=1,
+ max_buffer_length=MAX_PENDING_MESSAGES,
+ turn_in_flight=True,
+ )
+
+ monkeypatch.setattr(helpers_module, "queue_user_message", push)
+ request = asyncio.create_task(_queue_with_client_id())
+ await appended.wait()
+ request.cancel()
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+ request.cancel()
+ await asyncio.sleep(0)
+ published.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await request
+
+ accept.assert_awaited_once_with("sess-1", "scoped-msg-1", ANY)
+ release.assert_not_awaited()
+
+
+async def test_queue_pending_tells_a_copy_to_retry_while_the_first_is_unsettled(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The first copy is still being pushed, so this one can be told neither
+ that the message was taken nor that it was refused."""
+ _, accept, release = _mock_queue_claims(monkeypatch, claim_result="reserved")
+ queue_mock = AsyncMock()
+ monkeypatch.setattr(helpers_module, "queue_user_message", queue_mock)
+
+ with pytest.raises(HTTPException) as exc_info:
+ await _queue_with_client_id()
+
+ assert exc_info.value.status_code == 503
+ queue_mock.assert_not_awaited()
+ accept.assert_not_awaited()
+ release.assert_not_awaited()
+
+
+async def test_queue_pending_retransmit_is_accepted_when_the_count_lookup_fails(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """The buffer length is informational, so a Redis error reading it still
+ answers the copy as accepted rather than failing it."""
+ _mock_queue_claims(monkeypatch, claim_result="accepted")
+ monkeypatch.setattr(
+ helpers_module,
+ "peek_pending_count",
+ AsyncMock(side_effect=ConnectionError("down")),
+ )
+
+ result = await _queue_with_client_id()
+
+ assert result.turn_in_flight is True
+ assert result.buffer_length == 0
+
+
@pytest.mark.asyncio
async def test_check_pending_call_rate_at_limit(
monkeypatch: pytest.MonkeyPatch,
diff --git a/autogpt_platform/backend/backend/copilot/pending_messages.py b/autogpt_platform/backend/backend/copilot/pending_messages.py
index e6e0274ef8..3fe1e4b7dd 100644
--- a/autogpt_platform/backend/backend/copilot/pending_messages.py
+++ b/autogpt_platform/backend/backend/copilot/pending_messages.py
@@ -21,19 +21,26 @@ A hard cap of ``MAX_PENDING_MESSAGES`` per session prevents abuse. The
buffer is trimmed to the latest ``MAX_PENDING_MESSAGES`` on every push.
"""
+import asyncio
import json
import logging
import time
import uuid
-from typing import Any, cast
+from typing import Any, Literal, TypeVar, cast
from pydantic import BaseModel, Field, ValidationError
from backend.data.redis_client import get_redis_async
-from backend.data.redis_helpers import capped_rpush, capped_rpush_if_hash_field
+from backend.data.redis_helpers import (
+ capped_rpush,
+ capped_rpush_if_hash_field,
+ string_compare_and_set,
+)
logger = logging.getLogger(__name__)
+T = TypeVar("T")
+
# Per-session cap; typing faster than the copilot drains is already unusual.
MAX_PENDING_MESSAGES = 10
@@ -51,6 +58,24 @@ _PERSIST_QUEUE_KEY_PREFIX = "copilot:pending-persist:"
# message as a wake-up hint; the value itself is not meaningful.
_NOTIFY_PAYLOAD = "1"
+# Client idempotency keys (``StreamChatRequest.message_id``, already scoped to
+# user + session) of sends the chat API has accepted. A buffered message has
+# no ``ChatMessage`` row until a turn drains it, and the drained row gets a
+# fresh id, so the PK that dedupes a turn-start send cannot catch a
+# retransmit of a queued one: without this claim every copy was buffered
+# again and ran as another follow-up.
+_CLIENT_MESSAGE_KEY_PREFIX = "copilot:pending:client-msg:"
+# A reservation names the request holding it, so one that outlived its TTL
+# cannot accept or release the claim of the request that took over.
+_CLIENT_MESSAGE_RESERVED = "reserved:"
+_CLIENT_MESSAGE_ACCEPTED = "accepted"
+# A reservation covers scheduling a turn or pushing to the buffer, which takes
+# well under a second; the TTL only matters when the request dies midway.
+_CLIENT_MESSAGE_RESERVE_TTL_SECONDS = 30
+# How long a copy waits for the first copy's outcome before it gives up.
+_CLIENT_MESSAGE_WAIT_SECONDS = 10.0
+_CLIENT_MESSAGE_POLL_SECONDS = 0.1
+
class PendingMessageContext(BaseModel):
"""Structured page context attached to a pending message.
@@ -94,6 +119,165 @@ def _notify_channel(session_id: str) -> str:
return f"{_PENDING_CHANNEL_PREFIX}{session_id}"
+def _client_message_key(session_id: str, message_id: str) -> str:
+ return f"{_CLIENT_MESSAGE_KEY_PREFIX}{session_id}:{message_id}"
+
+
+ClientMessageClaim = Literal["claimed", "accepted", "reserved"]
+
+
+async def _settled_client_message(redis: Any, key: str, deadline: float) -> str | None:
+ """The key's state once no other request is still scheduling its send,
+ or ``"reserved"`` if that has not happened by *deadline*."""
+ while True:
+ state = await redis.get(key)
+ if not _is_reservation(state) or time.monotonic() >= deadline:
+ return state
+ await asyncio.sleep(_CLIENT_MESSAGE_POLL_SECONDS)
+
+
+def _is_reservation(state: str | None) -> bool:
+ return state is not None and state.startswith(_CLIENT_MESSAGE_RESERVED)
+
+
+async def claim_client_message(
+ session_id: str, message_id: str, owner: str
+) -> ClientMessageClaim:
+ """Reserve *message_id* for the request carrying it, identified by the
+ unique *owner* token it passes to accept or release the claim.
+
+ ``"claimed"``: the caller owns the send. It must call
+ :func:`accept_client_message` once the send is scheduled or queued, or
+ :func:`release_client_message` when it is refused. The reservation
+ expires on its own within seconds, so a request that dies midway
+ cannot block the id.
+
+ ``"accepted"``: the caller holds a retransmit of a message that is
+ queued, drained or persisted, and must not act on it again.
+
+ ``"reserved"``: another copy was still being scheduled when the wait ran
+ out, so the caller should tell the client to retry. A copy waits for
+ that outcome rather than being told it was accepted: the first copy can
+ still be refused, and then the send was never taken.
+
+ Fails open to ``"claimed"`` on a Redis error so dedup can never block a
+ send.
+ """
+ key = _client_message_key(session_id, message_id)
+ deadline = time.monotonic() + _CLIENT_MESSAGE_WAIT_SECONDS
+ try:
+ redis = await get_redis_async()
+ while True:
+ if await redis.set(
+ key,
+ _CLIENT_MESSAGE_RESERVED + owner,
+ nx=True,
+ ex=_CLIENT_MESSAGE_RESERVE_TTL_SECONDS,
+ ):
+ return "claimed"
+ state = await _settled_client_message(redis, key, deadline)
+ if _is_reservation(state):
+ return "reserved"
+ if state is not None:
+ return "accepted"
+ # The first copy was refused and gave the id back: take it.
+ except Exception as e:
+ logger.warning(
+ "pending_messages: client message claim failed for session=%s: %s",
+ session_id,
+ e,
+ )
+ return "claimed"
+
+
+async def finish_despite_cancellation(work: "asyncio.Future[T]") -> T:
+ """Wait for shielded cleanup *work* however many times the caller is
+ cancelled meanwhile; the caller re-raises its own cancellation after."""
+ while True:
+ try:
+ return await asyncio.shield(work)
+ except asyncio.CancelledError:
+ if work.done():
+ return work.result()
+
+
+async def accept_client_message(session_id: str, message_id: str, owner: str) -> None:
+ """Mark a claimed send as taken, so its retransmits are skipped. Also
+ when the reservation has lapsed, unless another request holds it now.
+
+ Finishes even if the caller is cancelled midway: the send has already
+ landed, and a reservation left to lapse would let a retry send it again.
+ """
+ accept = asyncio.ensure_future(
+ _accept_client_message(session_id, message_id, owner)
+ )
+ try:
+ await asyncio.shield(accept)
+ except asyncio.CancelledError:
+ await finish_despite_cancellation(accept)
+ raise
+
+
+async def _accept_client_message(session_id: str, message_id: str, owner: str) -> None:
+ try:
+ redis = await get_redis_async()
+ await string_compare_and_set(
+ redis,
+ _client_message_key(session_id, message_id),
+ expected=_CLIENT_MESSAGE_RESERVED + owner,
+ new=_CLIENT_MESSAGE_ACCEPTED,
+ ttl_seconds=_PENDING_TTL_SECONDS,
+ or_missing=True,
+ )
+ except Exception as e:
+ logger.warning(
+ "pending_messages: client message accept failed for session=%s: %s",
+ session_id,
+ e,
+ )
+
+
+async def client_message_state(
+ session_id: str, message_id: str
+) -> Literal["accepted", "reserved"] | None:
+ """Whether a send carrying *message_id* was taken, waiting as
+ :func:`claim_client_message` does while another copy is scheduling it."""
+ deadline = time.monotonic() + _CLIENT_MESSAGE_WAIT_SECONDS
+ try:
+ redis = await get_redis_async()
+ state = await _settled_client_message(
+ redis, _client_message_key(session_id, message_id), deadline
+ )
+ except Exception as e:
+ logger.warning(
+ "pending_messages: client message lookup failed for session=%s: %s",
+ session_id,
+ e,
+ )
+ return None
+ if state is None:
+ return None
+ return "reserved" if _is_reservation(state) else "accepted"
+
+
+async def release_client_message(session_id: str, message_id: str, owner: str) -> None:
+ """Drop a claim whose send was refused, so a genuine retry can land."""
+ try:
+ redis = await get_redis_async()
+ await string_compare_and_set(
+ redis,
+ _client_message_key(session_id, message_id),
+ expected=_CLIENT_MESSAGE_RESERVED + owner,
+ new=None,
+ )
+ except Exception as e:
+ logger.warning(
+ "pending_messages: client message release failed for session=%s: %s",
+ session_id,
+ e,
+ )
+
+
def _decode_redis_item(item: Any) -> str:
"""Decode a redis-py list item to str (handles ``bytes`` and ``str``)."""
return item.decode("utf-8") if isinstance(item, bytes) else str(item)
diff --git a/autogpt_platform/backend/backend/copilot/pending_messages_test.py b/autogpt_platform/backend/backend/copilot/pending_messages_test.py
index 16f03f1f97..5fb87573dc 100644
--- a/autogpt_platform/backend/backend/copilot/pending_messages_test.py
+++ b/autogpt_platform/backend/backend/copilot/pending_messages_test.py
@@ -8,6 +8,7 @@ and pull in the full app startup).
import asyncio
import json
from typing import Any
+from unittest.mock import AsyncMock
import pytest
@@ -16,7 +17,10 @@ from backend.copilot.pending_messages import (
MAX_PENDING_MESSAGES,
PendingMessage,
PendingMessageContext,
+ accept_client_message,
+ claim_client_message,
clear_pending_messages_unsafe,
+ client_message_state,
drain_and_format_for_injection,
drain_pending_for_persist,
drain_pending_messages,
@@ -25,6 +29,7 @@ from backend.copilot.pending_messages import (
peek_pending_count,
peek_pending_messages,
push_pending_message,
+ release_client_message,
stash_pending_for_persist,
)
@@ -37,6 +42,8 @@ class _FakeRedis:
# bytes when ``decode_responses=False``; the drain path must
# handle both and our tests exercise both.
self.lists: dict[str, list[str | bytes]] = {}
+ self.strings: dict[str, str] = {}
+ self.ttls: dict[str, int | None] = {}
self.published: list[tuple[str, str]] = []
async def rpush(self, key: str, *values: Any) -> int:
@@ -92,7 +99,43 @@ class _FakeRedis:
if key in self.lists:
del self.lists[key]
return 1
- return 0
+ return int(self.strings.pop(key, None) is not None)
+
+ async def set(
+ self, key: str, value: str, nx: bool = False, ex: int | None = None
+ ) -> bool | None:
+ if nx and key in self.strings:
+ return None
+ self.strings[key] = value
+ self.ttls[key] = ex
+ return True
+
+ async def get(self, key: str) -> str | None:
+ return self.strings.get(key)
+
+ async def eval(
+ self,
+ script: str,
+ numkeys: int,
+ key: str,
+ expected: str,
+ new: str,
+ ttl_seconds: int,
+ or_missing: str,
+ ) -> int:
+ """``string_compare_and_set``, the only script these tests reach."""
+ current = self.strings.get(key)
+ if current != expected and not (current is None and or_missing == "1"):
+ return 0
+ if new:
+ self.strings[key] = new
+ self.ttls[key] = int(ttl_seconds)
+ else:
+ self.strings.pop(key, None)
+ return 1
+
+ async def exists(self, key: str) -> int:
+ return int(key in self.strings or key in self.lists)
def pipeline(self, transaction: bool = True) -> "_FakePipeline":
# Returns a fake pipeline that records ops and replays them in
@@ -716,3 +759,203 @@ def test_buffer_and_session_meta_keys_share_cluster_slot() -> None:
f"CROSSSLOT regression: {buf!r} (slot {_redis_keyslot(buf)}) "
f"!= {meta!r} (slot {_redis_keyslot(meta)})"
)
+
+
+# ── Client message claims (SECRT-2695) ──────────────────────────────
+
+
+def short_claim_wait(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(pm_module, "_CLIENT_MESSAGE_WAIT_SECONDS", 0.05)
+ monkeypatch.setattr(pm_module, "_CLIENT_MESSAGE_POLL_SECONDS", 0.01)
+
+
+async def test_client_message_claim_is_granted_once(fake_redis: _FakeRedis) -> None:
+ """A retransmit of an accepted send must find the id already taken."""
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ await accept_client_message("sess-1", "msg-a", "first")
+ assert await claim_client_message("sess-1", "msg-a", "copy") == "accepted"
+ assert await client_message_state("sess-1", "msg-a") == "accepted"
+
+
+async def test_client_message_reservation_is_short_and_acceptance_long(
+ fake_redis: _FakeRedis,
+) -> None:
+ """A request that dies before settling its send must not block the id
+ for the hour an accepted send is remembered."""
+ key = pm_module._client_message_key("sess-1", "msg-a")
+
+ await claim_client_message("sess-1", "msg-a", "first")
+ assert fake_redis.ttls[key] == pm_module._CLIENT_MESSAGE_RESERVE_TTL_SECONDS
+ await accept_client_message("sess-1", "msg-a", "first")
+ assert fake_redis.ttls[key] == pm_module._PENDING_TTL_SECONDS
+
+
+async def test_client_message_claims_are_per_id_and_session(
+ fake_redis: _FakeRedis,
+) -> None:
+ assert await claim_client_message("sess-1", "msg-a", "o1") == "claimed"
+ assert await claim_client_message("sess-1", "msg-b", "o2") == "claimed"
+ assert await claim_client_message("sess-2", "msg-a", "o3") == "claimed"
+
+
+async def test_released_client_message_can_be_claimed_again(
+ fake_redis: _FakeRedis,
+) -> None:
+ """A refused send drops its claim so the same id can land on retry."""
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ await release_client_message("sess-1", "msg-a", "first")
+ assert await client_message_state("sess-1", "msg-a") is None
+ assert await claim_client_message("sess-1", "msg-a", "retry") == "claimed"
+
+
+async def test_lapsed_claimant_cannot_settle_its_successors_claim(
+ fake_redis: _FakeRedis,
+) -> None:
+ """The first request outlived its reservation and another took the id
+ over: the first can neither release nor accept the successor's claim."""
+ key = pm_module._client_message_key("sess-1", "msg-a")
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ del fake_redis.strings[key] # the reservation expired
+ assert await claim_client_message("sess-1", "msg-a", "successor") == "claimed"
+
+ await release_client_message("sess-1", "msg-a", "first")
+ await accept_client_message("sess-1", "msg-a", "first")
+
+ assert fake_redis.strings[key] == "reserved:successor"
+
+
+async def test_lapsed_claimant_still_records_its_accepted_send(
+ fake_redis: _FakeRedis,
+) -> None:
+ """Nobody took the id over, so the send is marked accepted anyway."""
+ key = pm_module._client_message_key("sess-1", "msg-a")
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ del fake_redis.strings[key] # the reservation expired
+
+ await accept_client_message("sess-1", "msg-a", "first")
+
+ assert await client_message_state("sess-1", "msg-a") == "accepted"
+
+
+async def test_accepting_a_claim_finishes_when_the_request_is_cancelled(
+ fake_redis: _FakeRedis, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """The send already landed, so a cancelled request must still record it:
+ a reservation left to lapse would let a retry send it again."""
+ key = pm_module._client_message_key("sess-1", "msg-a")
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ accepting = asyncio.Event()
+ finish = asyncio.Event()
+ settle = fake_redis.eval
+
+ async def slow_eval(*args: Any) -> int:
+ accepting.set()
+ await finish.wait()
+ return await settle(*args)
+
+ monkeypatch.setattr(fake_redis, "eval", slow_eval)
+ request = asyncio.create_task(accept_client_message("sess-1", "msg-a", "first"))
+ await accepting.wait()
+
+ request.cancel()
+ await asyncio.sleep(0)
+ finish.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await request
+ assert fake_redis.strings[key] == "accepted"
+
+
+async def test_accepting_a_claim_finishes_when_the_request_is_cancelled_twice(
+ fake_redis: _FakeRedis, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A second cancellation while the accept is finishing must not cut it
+ short either, or the reservation is left to lapse."""
+ key = pm_module._client_message_key("sess-1", "msg-a")
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ accepting = asyncio.Event()
+ finish = asyncio.Event()
+ settle = fake_redis.eval
+
+ async def slow_eval(*args: Any) -> int:
+ accepting.set()
+ await finish.wait()
+ return await settle(*args)
+
+ monkeypatch.setattr(fake_redis, "eval", slow_eval)
+ request = asyncio.create_task(accept_client_message("sess-1", "msg-a", "first"))
+ await accepting.wait()
+
+ request.cancel()
+ await asyncio.sleep(0)
+ await asyncio.sleep(0)
+ request.cancel()
+ await asyncio.sleep(0)
+ finish.set()
+
+ with pytest.raises(asyncio.CancelledError):
+ await request
+ assert fake_redis.strings[key] == "accepted"
+
+
+async def test_copy_during_reservation_waits_for_acceptance(
+ fake_redis: _FakeRedis, short_claim_wait: None
+) -> None:
+ """A copy is not told the send was taken until it has been."""
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ copy = asyncio.create_task(claim_client_message("sess-1", "msg-a", "copy"))
+ await asyncio.sleep(0.02)
+ assert not copy.done()
+
+ await accept_client_message("sess-1", "msg-a", "first")
+
+ assert await copy == "accepted"
+
+
+async def test_copy_during_reservation_takes_the_id_when_the_first_is_refused(
+ fake_redis: _FakeRedis, short_claim_wait: None
+) -> None:
+ """The first copy was refused, so the send was never taken: the copy
+ carries it instead of being answered as a duplicate."""
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ copy = asyncio.create_task(claim_client_message("sess-1", "msg-a", "copy"))
+ await asyncio.sleep(0.02)
+
+ await release_client_message("sess-1", "msg-a", "first")
+
+ assert await copy == "claimed"
+
+
+async def test_copy_gives_up_when_the_reservation_outlasts_the_wait(
+ fake_redis: _FakeRedis, short_claim_wait: None
+) -> None:
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+
+ assert await claim_client_message("sess-1", "msg-a", "copy") == "reserved"
+ assert await client_message_state("sess-1", "msg-a") == "reserved"
+
+
+async def test_client_message_claims_fail_open_on_redis_error(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ """Dedup is best effort: a Redis blip must never block a send."""
+ monkeypatch.setattr(
+ pm_module, "get_redis_async", AsyncMock(side_effect=ConnectionError("down"))
+ )
+ assert await claim_client_message("sess-1", "msg-a", "first") == "claimed"
+ assert await client_message_state("sess-1", "msg-a") is None
+ await accept_client_message("sess-1", "msg-a", "first")
+ await release_client_message("sess-1", "msg-a", "first")
diff --git a/autogpt_platform/backend/backend/data/redis_helpers.py b/autogpt_platform/backend/backend/data/redis_helpers.py
index 71059eb275..c617400b61 100644
--- a/autogpt_platform/backend/backend/data/redis_helpers.py
+++ b/autogpt_platform/backend/backend/data/redis_helpers.py
@@ -13,6 +13,8 @@ patterns we actually use into a single place:
- :func:`hash_compare_and_set` — set a hash field only if its current
value matches an expected one. Genuinely needs Lua because the
condition depends on the current value (pipeline can't branch).
+- :func:`string_compare_and_set` — the same for a plain string key, with
+ delete as the "new value" option (ownership-checked release/handover).
Everything sharable lives here. If a new Lua script is tempting in
application code, add a helper here first — callers should not touch
@@ -63,6 +65,26 @@ return 1
# ARGV[3] list value
# ARGV[4] max list length
# ARGV[5] list TTL seconds
+# Compare-and-set on a string key. Returns 1 if swapped, 0 otherwise.
+#
+# KEYS[1] key
+# ARGV[1] expected current value
+# ARGV[2] new value, or "" to delete the key
+# ARGV[3] TTL in seconds for the new value
+# ARGV[4] "1" to also swap when the key is missing
+_STRING_CAS_LUA = """
+local current = redis.call('GET', KEYS[1])
+if current ~= ARGV[1] and not (current == false and ARGV[4] == '1') then
+ return 0
+end
+if ARGV[2] == '' then
+ redis.call('DEL', KEYS[1])
+else
+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])
+end
+return 1
+"""
+
_GATED_CAPPED_RPUSH_LUA = """
local current = redis.call('HGET', KEYS[1], ARGV[1])
if current ~= ARGV[2] then
@@ -433,3 +455,34 @@ async def hash_compare_and_set(
redis.eval(_HASH_CAS_LUA, 1, key, field, expected, new, *(guard or ())),
)
return int(result) == 1
+
+
+async def string_compare_and_set(
+ redis: AsyncRedisClient,
+ key: str,
+ *,
+ expected: str,
+ new: str | None,
+ ttl_seconds: int = 0,
+ or_missing: bool = False,
+) -> bool:
+ """Atomically replace *key* with *new* (``None`` deletes it) iff it still
+ holds *expected*, or is missing and *or_missing* is set.
+
+ Returns ``True`` if the swap happened. Use it when a value names its
+ owner, so a holder whose key expired and was taken over cannot overwrite
+ or delete its successor's value.
+ """
+ result = await cast(
+ "Any",
+ redis.eval(
+ _STRING_CAS_LUA,
+ 1,
+ key,
+ expected,
+ new or "",
+ ttl_seconds,
+ "1" if or_missing else "0",
+ ),
+ )
+ return int(result) == 1
diff --git a/autogpt_platform/backend/backend/data/redis_helpers_test.py b/autogpt_platform/backend/backend/data/redis_helpers_test.py
index 9ce31d7faf..edaff125ac 100644
--- a/autogpt_platform/backend/backend/data/redis_helpers_test.py
+++ b/autogpt_platform/backend/backend/data/redis_helpers_test.py
@@ -17,6 +17,7 @@ from backend.data.redis_helpers import (
hash_compare_and_set,
incr_with_ttl,
incr_with_ttl_sync,
+ string_compare_and_set,
)
# ── Fake Redis + pipeline ──────────────────────────────────────────────
@@ -87,6 +88,19 @@ class _Fake:
await self.expire(list_key, int(ttl_seconds))
return await self.llen(list_key)
+ if "'GET'" in script:
+ # ``string_compare_and_set`` shape.
+ key, expected, new, ttl_seconds, or_missing = args[:5]
+ current = self.strings.get(key)
+ if current != expected and not (current is None and or_missing == "1"):
+ return 0
+ if new:
+ self.strings[key] = new
+ self.ttls[key] = int(ttl_seconds)
+ else:
+ self.strings.pop(key, None)
+ return 1
+
# ``hash_compare_and_set`` shape (numkeys == 1).
key, field, expected, new = args[0], args[1], args[2], args[3]
h = self.hashes.setdefault(key, {})
@@ -298,6 +312,57 @@ async def test_hash_cas_no_swap_when_expected_differs() -> None:
assert r.hashes["meta"]["status"] == "completed"
+# ── string_compare_and_set ─────────────────────────────────────────────
+
+
+async def test_string_cas_swaps_and_sets_the_ttl_when_expected_matches() -> None:
+ r = _Fake()
+ r.strings["claim"] = "reserved:a"
+ swapped = await string_compare_and_set(
+ r, "claim", expected="reserved:a", new="accepted", ttl_seconds=60 # type: ignore[arg-type]
+ )
+ assert swapped is True
+ assert r.strings["claim"] == "accepted"
+ assert r.ttls["claim"] == 60
+
+
+async def test_string_cas_deletes_when_new_is_none() -> None:
+ r = _Fake()
+ r.strings["claim"] = "reserved:a"
+ swapped = await string_compare_and_set(
+ r, "claim", expected="reserved:a", new=None # type: ignore[arg-type]
+ )
+ assert swapped is True
+ assert "claim" not in r.strings
+
+
+async def test_string_cas_leaves_another_holders_value_alone() -> None:
+ r = _Fake()
+ r.strings["claim"] = "reserved:b"
+ assert not await string_compare_and_set(
+ r, "claim", expected="reserved:a", new=None # type: ignore[arg-type]
+ )
+ assert not await string_compare_and_set(
+ r, "claim", expected="reserved:a", new="accepted", or_missing=True # type: ignore[arg-type]
+ )
+ assert r.strings["claim"] == "reserved:b"
+
+
+async def test_string_cas_or_missing_sets_a_missing_key() -> None:
+ r = _Fake()
+ assert not await string_compare_and_set(
+ r, "claim", expected="reserved:a", new="accepted" # type: ignore[arg-type]
+ )
+ assert await string_compare_and_set(
+ r, "claim", expected="reserved:a", new="accepted", or_missing=True # type: ignore[arg-type]
+ )
+ assert r.strings["claim"] == "accepted"
+
+
# ── claim_batch_dispatch_atomic ───────────────────────────────────────
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts
index 7a4df0def5..a8081d9d84 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/__tests__/queueFollowUpMessage.test.ts
@@ -60,9 +60,36 @@ describe("queueFollowUpMessage", () => {
message: "hello",
context: null,
file_ids: null,
+ message_id: expect.stringMatching(
+ /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
+ ),
});
});
+ it("sends a fresh idempotency key for every queued message", async () => {
+ const fetchMock = vi.mocked(global.fetch);
+ for (let i = 0; i < 2; i++) {
+ fetchMock.mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ buffer_length: i + 1,
+ max_buffer_length: 10,
+ turn_in_flight: true,
+ }),
+ { status: 200, headers: { "Content-Type": "application/json" } },
+ ),
+ );
+ }
+
+ await queueFollowUpMessage("sess-1", "same text");
+ await queueFollowUpMessage("sess-1", "same text");
+
+ const ids = fetchMock.mock.calls.map(
+ ([, init]) => JSON.parse(init?.body as string).message_id,
+ );
+ expect(ids[0]).not.toEqual(ids[1]);
+ });
+
it("treats a 200 inactive response as not active", async () => {
const fetchMock = vi.mocked(global.fetch);
fetchMock.mockResolvedValueOnce(
diff --git a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts
index 0267369902..59867ef371 100644
--- a/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts
+++ b/autogpt_platform/frontend/src/app/(platform)/copilot/helpers/queueFollowUpMessage.ts
@@ -1,4 +1,5 @@
import { environment } from "@/services/environment";
+import { v4 as uuidv4 } from "uuid";
import { getCopilotAuthHeaders } from "../helpers";
@@ -34,6 +35,10 @@ export async function queueFollowUpMessage(
message,
context: null,
file_ids: null,
+ // Idempotency key: a retransmit of this request is queued only once.
+ // Options force uuid's getRandomValues path, which (unlike
+ // crypto.randomUUID) exists on plain-HTTP LAN origins.
+ message_id: uuidv4({}),
}),
});
diff --git a/autogpt_platform/frontend/src/app/api/openapi.json b/autogpt_platform/frontend/src/app/api/openapi.json
index bfccd3c7b0..4fd7d69f7c 100644
--- a/autogpt_platform/frontend/src/app/api/openapi.json
+++ b/autogpt_platform/frontend/src/app/api/openapi.json
@@ -28548,6 +28548,14 @@
{ "type": "null" }
],
"title": "Folder Ids"
+ },
+ "message_id": {
+ "anyOf": [
+ { "type": "string", "maxLength": 64 },
+ { "type": "null" }
+ ],
+ "title": "Message Id",
+ "description": "Optional per-send UUID, scoped like ``StreamChatRequest.message_id``. A retransmit of a follow-up already queued is answered as accepted without queueing it again."
}
},
"type": "object",
@@ -31023,7 +31031,7 @@
{ "type": "null" }
],
"title": "Message Id",
- "description": "Optional per-click UUID generated by the frontend. The server scopes it to the authenticated user and session before using the result as ``ChatMessage.id``. Frontend / network / RMQ-redelivery retransmits of the same logical send reuse the key, so the Postgres unique-constraint on the resulting PK is the atomic dedup primitive. A duplicate INSERT returns a subscribe-only response without creating a parallel turn. Distinct user clicks (even with identical text) MUST send different ids — the frontend's per-click ``crypto.randomUUID()`` guarantees that."
+ "description": "Optional per-click UUID generated by the frontend. The server scopes it to the authenticated user and session before using the result as ``ChatMessage.id``. Frontend / network / RMQ-redelivery retransmits of the same logical send reuse the key, and the server claims it once: a retransmit neither starts a parallel turn nor, while a turn is running, queues the message again, and gets a subscribe-only response. Distinct user clicks (even with identical text) MUST send different ids — the frontend's per-click ``crypto.randomUUID()`` guarantees that."
},
"expert_kickoff": {
"type": "boolean",