Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🧷 fix: Fence Server-Admitted Queue Boundaries (#15393)

Danny Avilaf0f82ec5fa85
diff --git a/client/src/components/Chat/Input/PendingSteerChips.tsx b/client/src/components/Chat/Input/PendingSteerChips.tsx
index eef0566152..3e21c69f3b 100644
--- a/client/src/components/Chat/Input/PendingSteerChips.tsx
+++ b/client/src/components/Chat/Input/PendingSteerChips.tsx
@@ -103,8 +103,18 @@ function QueuedRow({
const quoteCount = message.quotes?.length ?? 0;
const isRecovered = message.recoverySteerId != null;
const isRejected = message.server?.status === 'rejected';
+ const isIndeterminate = message.server?.status === 'indeterminate';
const isUnconfirmed =
message.server?.status === 'uncertain' && message.server.reconciliationExpired === true;
+ let statusLabel:
+ | 'com_ui_queued_turn_reconciliation_required'
+ | 'com_ui_steer_delivery_unconfirmed'
+ | 'com_ui_queued_turn_failed' = 'com_ui_queued_turn_failed';
+ if (isIndeterminate) {
+ statusLabel = 'com_ui_queued_turn_reconciliation_required';
+ } else if (isUnconfirmed) {
+ statusLabel = 'com_ui_steer_delivery_unconfirmed';
+ }
const requiresDiscard = isRecovered || message.server?.id != null;
const serverActionable =
message.server == null ||
@@ -198,7 +208,7 @@ function QueuedRow({
return (
<div role="listitem" className={ROW_CLASS} data-testid="queued-message-row">
- {isRejected || isUnconfirmed ? (
+ {isRejected || isUnconfirmed || isIndeterminate ? (
<TriangleAlert className="h-4 w-4 shrink-0 text-text-warning" aria-hidden="true" />
) : (
<Clock className="h-4 w-4 shrink-0 text-cyan-500" aria-hidden="true" />
@@ -216,12 +226,8 @@ function QueuedRow({
0: String(fileCount),
})}
/>
- {(isRejected || isUnconfirmed) && (
- <span className="shrink-0 text-xs text-text-warning">
- {localize(
- isUnconfirmed ? 'com_ui_steer_delivery_unconfirmed' : 'com_ui_queued_turn_failed',
- )}
- </span>
+ {(isRejected || isUnconfirmed || isIndeterminate) && (
+ <span className="shrink-0 text-xs text-text-warning">{localize(statusLabel)}</span>
)}
{showPrimary && (
<button
diff --git a/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx b/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
index 09b972c012..8a28e5ed11 100644
--- a/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
+++ b/client/src/components/Chat/Input/__tests__/PendingSteerChips.test.tsx
@@ -181,6 +181,25 @@ describe('PendingSteerChips — terminal server state', () => {
expect(screen.getByText('com_ui_queued_turn_failed')).toBeInTheDocument();
});
+ it('shows indeterminate admission as non-actionable reconciliation work', () => {
+ renderChips([
+ {
+ id: 'indeterminate-turn',
+ text: 'external result unknown',
+ createdAt: 1,
+ server: {
+ id: 'server-indeterminate-1',
+ status: 'indeterminate',
+ errorCode: 'ADMISSION_INDETERMINATE',
+ },
+ },
+ ]);
+
+ expect(screen.getByText('com_ui_queued_turn_reconciliation_required')).toBeInTheDocument();
+ expect(screen.getByLabelText('com_ui_remove_queued')).toBeDisabled();
+ expect(screen.queryByText('com_ui_send_now')).toBeNull();
+ });
+
it('dismisses an expired ambiguous receipt without restoring or resending it', () => {
renderChips([
{
diff --git a/client/src/data-provider/SSE/__tests__/queuedTurns.test.ts b/client/src/data-provider/SSE/__tests__/queuedTurns.test.ts
index 1ea5de10fe..be7931d9be 100644
--- a/client/src/data-provider/SSE/__tests__/queuedTurns.test.ts
+++ b/client/src/data-provider/SSE/__tests__/queuedTurns.test.ts
@@ -1,6 +1,18 @@
+import { act, renderHook, waitFor } from '@testing-library/react';
+
const mockListAgentQueuedTurns = jest.fn();
const mockEnqueueAgentQueuedTurn = jest.fn();
const mockCancelAgentQueuedTurn = jest.fn();
+const mockRefetch = jest.fn();
+const mockCancelQueries = jest.fn().mockResolvedValue(undefined);
+const mockQueryClient = { cancelQueries: mockCancelQueries };
+const mockUseQuery = jest.fn((options: unknown) => ({ options, refetch: mockRefetch }));
+
+jest.mock('@tanstack/react-query', () => ({
+ useQuery: (options: unknown) => mockUseQuery(options),
+ useMutation: jest.fn(),
+ useQueryClient: () => mockQueryClient,
+}));
jest.mock('librechat-data-provider', () => ({
QueryKeys: { agentQueuedTurns: 'agentQueuedTurns' },
@@ -21,6 +33,8 @@ import {
fetchAgentQueuedTurns,
isDefiniteQueuedTurnRejection,
isDefiniteQueuedTurnsUnsupported,
+ shouldPollAgentQueuedTurns,
+ useAgentQueuedTurns,
} from '../queuedTurns';
describe('Agent queued-turn data adapter', () => {
@@ -116,4 +130,87 @@ describe('Agent queued-turn data adapter', () => {
expect(isDefiniteQueuedTurnRejection(error)).toBe(false);
},
);
+
+ it('stops background polling for a fail-closed indeterminate claim', () => {
+ expect(
+ shouldPollAgentQueuedTurns([
+ {
+ status: 'claimed',
+ failure: { code: 'ADMISSION_INDETERMINATE' },
+ },
+ ]),
+ ).toBe(false);
+ expect(shouldPollAgentQueuedTurns([{ status: 'claimed' }])).toBe(true);
+ expect(shouldPollAgentQueuedTurns([{ status: 'queued' }])).toBe(true);
+ });
+
+ it('refreshes stopped reconciliation work on every mount and focus', () => {
+ renderHook(() => useAgentQueuedTurns('conversation/one', true));
+
+ expect(mockUseQuery).toHaveBeenCalledWith(
+ expect.objectContaining({
+ refetchOnMount: 'always',
+ refetchOnWindowFocus: 'always',
+ }),
+ );
+ });
+
+ it('keeps one cache authority and refetches when known receipt ids change', async () => {
+ const rendered = renderHook(
+ ({ ids }: { ids: string[] }) => useAgentQueuedTurns('conversation/one', true, ids),
+ { initialProps: { ids: ['request-one'] } },
+ );
+
+ expect(mockUseQuery).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ queryKey: ['agentQueuedTurns', 'conversation/one'],
+ }),
+ );
+ expect(mockRefetch).not.toHaveBeenCalled();
+
+ rendered.rerender({ ids: [] });
+
+ await waitFor(() =>
+ expect(mockCancelQueries).toHaveBeenCalledWith({
+ queryKey: ['agentQueuedTurns', 'conversation/one'],
+ exact: true,
+ }),
+ );
+ await waitFor(() =>
+ expect(mockRefetch).toHaveBeenCalledWith({
+ cancelRefetch: true,
+ }),
+ );
+ expect(mockCancelQueries.mock.invocationCallOrder[0]).toBeLessThan(
+ mockRefetch.mock.invocationCallOrder[0],
+ );
+ expect(mockUseQuery).toHaveBeenLastCalledWith(
+ expect.objectContaining({
+ queryKey: ['agentQueuedTurns', 'conversation/one'],
+ }),
+ );
+ });
+
+ it('lets only the newest identity projection refetch after cancellation', async () => {
+ let releaseFirstCancellation: (() => void) | undefined;
+ mockCancelQueries
+ .mockImplementationOnce(
+ () =>
+ new Promise<void>((resolve) => {
+ releaseFirstCancellation = resolve;
+ }),
+ )
+ .mockResolvedValueOnce(undefined);
+ const rendered = renderHook(
+ ({ ids }: { ids: string[] }) => useAgentQueuedTurns('conversation/one', true, ids),
+ { initialProps: { ids: ['request-one'] } },
+ );
+
+ rendered.rerender({ ids: ['request-two'] });
+ rendered.rerender({ ids: ['request-three'] });
+
+ await waitFor(() => expect(mockRefetch).toHaveBeenCalledTimes(1));
+ await act(async () => releaseFirstCancellation?.());
+ expect(mockRefetch).toHaveBeenCalledTimes(1);
+ });
});
diff --git a/client/src/data-provider/SSE/queuedTurns.ts b/client/src/data-provider/SSE/queuedTurns.ts
index fb781c98f8..df991619aa 100644
--- a/client/src/data-provider/SSE/queuedTurns.ts
+++ b/client/src/data-provider/SSE/queuedTurns.ts
@@ -1,3 +1,4 @@
+import { useEffect, useRef } from 'react';
import { useMutation, useQuery, useQueryClient } from '@tanstack/react-query';
import { dataService, MutationKeys, QueryKeys } from 'librechat-data-provider';
import type {
@@ -8,10 +9,12 @@ import type {
export type AgentQueuedTurnReceipt = TAgentQueuedTurnReceipt;
export type EnqueueAgentQueuedTurnRequest = TEnqueueAgentQueuedTurnRequest;
-export const agentQueuedTurnsQueryKey = (
- conversationId: string,
- clientRequestIds: readonly string[] = [],
-) => [QueryKeys.agentQueuedTurns, conversationId, ...clientRequestIds] as const;
+/** Keep one cache authority for a conversation. Known receipt ids change the
+ * server projection, not the identity of the durable queue. Giving every id
+ * set its own key can resurrect an older cached projection after an id is
+ * retired. */
+export const agentQueuedTurnsQueryKey = (conversationId: string) =>
+ [QueryKeys.agentQueuedTurns, conversationId] as const;
function queuedTurnErrorResponse(error: unknown): { status?: number; code?: string } {
const response = (
@@ -72,29 +75,85 @@ export async function cancelAgentQueuedTurn(input: {
return response.receipt;
}
+/** Indeterminate admission deliberately retains its durable claim until exact
+ * source evidence arrives. It can still be refreshed on mount/focus, but a
+ * two-second loop cannot resolve it and disguises permanent quarantine as
+ * ordinary progress. */
+export function shouldPollAgentQueuedTurns(
+ receipts: unknown,
+ reconcileUntil?: number,
+ observedAt = Date.now(),
+): boolean {
+ return (
+ (reconcileUntil != null && observedAt < reconcileUntil) ||
+ (Array.isArray(receipts) &&
+ receipts.some(
+ (item: AgentQueuedTurnReceipt) =>
+ item.status === 'queued' ||
+ (item.status === 'claimed' && item.failure?.code !== 'ADMISSION_INDETERMINATE'),
+ ))
+ );
+}
+
export function useAgentQueuedTurns(
conversationId: string,
enabled: boolean,
clientRequestIds: string[] = [],
reconcileUntil?: number,
) {
+ const queryClient = useQueryClient();
const knownIds = [...new Set(clientRequestIds)].sort();
- return useQuery({
- queryKey: agentQueuedTurnsQueryKey(conversationId, knownIds),
+ const knownIdsSignature = knownIds.join('\u0000');
+ const previousRequest = useRef({ conversationId, knownIdsSignature });
+ const query = useQuery({
+ queryKey: agentQueuedTurnsQueryKey(conversationId),
queryFn: () => fetchAgentQueuedTurns(conversationId, knownIds),
enabled: enabled && conversationId.length > 0,
staleTime: 1_000,
- refetchOnMount: true,
- refetchOnWindowFocus: true,
- refetchInterval: (receipts) => {
- return (reconcileUntil != null && Date.now() < reconcileUntil) ||
- (Array.isArray(receipts) &&
- receipts.some((item) => item.status === 'queued' || item.status === 'claimed'))
- ? 2_000
- : false;
- },
+ /** A stopped indeterminate row can receive exact proof at any time. Its
+ * cache may still be inside `staleTime`, so mount/focus must bypass the
+ * ordinary freshness gate rather than waiting for another user cycle. */
+ refetchOnMount: 'always',
+ refetchOnWindowFocus: 'always',
+ refetchInterval: (receipts) =>
+ shouldPollAgentQueuedTurns(receipts, reconcileUntil) ? 2_000 : false,
retry: false,
});
+ const { refetch } = query;
+
+ useEffect(() => {
+ const previous = previousRequest.current;
+ previousRequest.current = { conversationId, knownIdsSignature };
+ if (
+ !enabled ||
+ conversationId.length === 0 ||
+ previous.conversationId !== conversationId ||
+ previous.knownIdsSignature === knownIdsSignature
+ ) {
+ return;
+ }
+
+ let superseded = false;
+ const refreshProjection = async () => {
+ /** React Query cannot replace an initial fetch that has no cached data
+ * through `refetch` alone. Cancel it explicitly so the next request
+ * captures the current reconciliation identities. */
+ await queryClient.cancelQueries({
+ queryKey: agentQueuedTurnsQueryKey(conversationId),
+ exact: true,
+ });
+ if (!superseded) {
+ await refetch({ cancelRefetch: true });
+ }
+ };
+ void refreshProjection();
+
+ return () => {
+ superseded = true;
+ };
+ }, [conversationId, enabled, knownIdsSignature, queryClient, refetch]);
+
+ return query;
}
export function useEnqueueAgentQueuedTurnMutation() {
diff --git a/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx b/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
index 762a5f692e..232c79c993 100644
--- a/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useQueueDrain.spec.tsx
@@ -3,7 +3,12 @@ import { Constants } from 'librechat-data-provider';
import { act, renderHook, waitFor } from '@testing-library/react';
import { QueryClient, QueryClientProvider } from '@tanstack/react-query';
import { RecoilRoot, useRecoilValue, useSetRecoilState, type MutableSnapshot } from 'recoil';
-import type { DrainAfterAbort, RunEnd, QueuedMessage } from '~/store/families';
+import type {
+ DrainAfterAbort,
+ RunEnd,
+ QueuedMessage,
+ SettledQueuedTurnReceipt,
+} from '~/store/families';
import useQueueDrain from '../useQueueDrain';
import store from '~/store';
@@ -26,8 +31,12 @@ function setup(
setIsSubmitting?: (value: boolean) => void;
setQueue?: (value: QueuedMessage[]) => void;
setNewConvoQueue?: (value: QueuedMessage[]) => void;
+ setSettledReceipts?: (value: SettledQueuedTurnReceipt[]) => void;
setInterruptFlag?: (value: DrainAfterAbort | false) => void;
- queueRef?: { current: QueuedMessage[] };
+ queue?: QueuedMessage[];
+ newConvoQueue?: QueuedMessage[];
+ settledReceipts?: SettledQueuedTurnReceipt[];
+ runEnd?: RunEnd | null;
} = {};
function Harness() {
@@ -37,7 +46,14 @@ function setup(
setters.setNewConvoQueue = useSetRecoilState(
store.queuedMessagesByConvoId(Constants.NEW_CONVO),
);
+ setters.setSettledReceipts = useSetRecoilState(
+ store.settledQueuedTurnReceiptsByConvoId(CONVO_ID),
+ );
setters.setInterruptFlag = useSetRecoilState(store.drainAfterAbortByIndex(INDEX));
+ setters.queue = useRecoilValue(store.queuedMessagesByConvoId(CONVO_ID));
+ setters.newConvoQueue = useRecoilValue(store.queuedMessagesByConvoId(Constants.NEW_CONVO));
+ setters.settledReceipts = useRecoilValue(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID));
+ setters.runEnd = useRecoilValue(store.runEndByIndex(INDEX));
useQueueDrain(INDEX, activeConversationId, ask);
return null;
}
@@ -150,6 +166,191 @@ describe('useQueueDrain', () => {
expect(ask).toHaveBeenCalledWith({ text: 'legacy successor' }, emptyOverrides);
});
+ it('reacts to a settled admission while another server row still owns the queue', async () => {
+ const admittedServer: QueuedMessage = {
+ ...queuedMessage('q-server-1', 'admitted server-owned turn'),
+ server: { id: 'server-queue-1', status: 'claimed', revision: 1 },
+ };
+ const remainingServer: QueuedMessage = {
+ ...queuedMessage('q-server-2', 'later server-owned turn'),
+ server: { id: 'server-queue-2', status: 'queued', revision: 2 },
+ };
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [admittedServer, remainingServer]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 41 }));
+ });
+ await new Promise((resolve) => setTimeout(resolve, 25));
+ expect(setters.runEnd).not.toBeNull();
+
+ act(() => {
+ setters.setQueue!([remainingServer]);
+ setters.setSettledReceipts!([
+ {
+ clientRequestId: 'admitted-request-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ await waitFor(() => expect(setters.runEnd).toBeNull());
+ expect(ask).not.toHaveBeenCalled();
+ expect(setters.settledReceipts).toEqual([]);
+ expect(setters.queue).toEqual([remainingServer]);
+ });
+
+ it('discards a predecessor boundary already consumed by server admission', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ queuedMessage('q-local', 'must wait for the admitted run'),
+ ]);
+ set(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID), [
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 41 }));
+ });
+
+ await waitFor(() => expect(setters.runEnd).toBeNull());
+ expect(ask).not.toHaveBeenCalled();
+ expect(setters.settledReceipts).toEqual([]);
+ });
+
+ it('consumes one admission when separate receipts share the same predecessor epoch', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ queuedMessage('q-local', 'wait for both admitted runs'),
+ ]);
+ set(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID), [
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ {
+ clientRequestId: 'admission-2',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 41 }));
+ });
+
+ await waitFor(() =>
+ expect(setters.settledReceipts).toEqual([
+ {
+ clientRequestId: 'admission-2',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]),
+ );
+ expect(ask).not.toHaveBeenCalled();
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 41, endedAt: Date.now() + 1 }));
+ });
+
+ await waitFor(() => expect(setters.settledReceipts).toEqual([]));
+ expect(ask).not.toHaveBeenCalled();
+ });
+
+ it('migrates the NEW_CONVO queue before discarding a consumed predecessor boundary', async () => {
+ const queuedBeforeResolution = queuedMessage('q-new', 'wait for the admitted successor');
+ const queuedAfterResolution = queuedMessage('q-resolved', 'still ordered after migration');
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(Constants.NEW_CONVO), [queuedBeforeResolution]);
+ set(store.queuedMessagesByConvoId(CONVO_ID), [queuedAfterResolution]);
+ set(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID), [
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(
+ runEnd({
+ generationCreatedAt: 41,
+ startedAsNewConvo: true,
+ }),
+ );
+ });
+
+ await waitFor(() => expect(setters.runEnd).toBeNull());
+ expect(ask).not.toHaveBeenCalled();
+ expect(setters.newConvoQueue).toEqual([]);
+ expect(setters.queue).toEqual([queuedBeforeResolution, queuedAfterResolution]);
+ });
+
+ it('lets the admitted successor terminal boundary release the next turn', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ queuedMessage('q-local', 'send after the admitted run'),
+ ]);
+ set(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID), [
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 42 }));
+ });
+
+ await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
+ expect(ask).toHaveBeenCalledWith({ text: 'send after the admitted run' }, emptyOverrides);
+ });
+
+ it('does not interpret a consumed predecessor as a timestamp range', async () => {
+ const { ask, setters } = setup(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ queuedMessage('q-local', 'send after the lower-clock successor'),
+ ]);
+ set(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID), [
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 42,
+ },
+ ]);
+ });
+
+ act(() => {
+ setters.setRunEnd!(runEnd({ generationCreatedAt: 41 }));
+ });
+
+ await waitFor(() => expect(ask).toHaveBeenCalledTimes(1));
+ expect(ask).toHaveBeenCalledWith(
+ { text: 'send after the lower-clock successor' },
+ emptyOverrides,
+ );
+ expect(setters.settledReceipts).toEqual([
+ {
+ clientRequestId: 'admission-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 42,
+ },
+ ]);
+ });
+
it('parks a mismatched signal instead of draining into the wrong conversation', async () => {
const { ask, setters } = setup(({ set }) => {
set(store.queuedMessagesByConvoId(CONVO_ID), [queuedMessage('q1', 'stay put')]);
diff --git a/client/src/hooks/Chat/__tests__/useSteering.spec.tsx b/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
index 0fa405366d..af68491473 100644
--- a/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
+++ b/client/src/hooks/Chat/__tests__/useSteering.spec.tsx
@@ -4,6 +4,7 @@ import { RecoilRoot, useRecoilValue, useSetRecoilState, type MutableSnapshot } f
import { Constants, ContentTypes, EModelEndpoint, LocalStorageKeys } from 'librechat-data-provider';
import type { TConversation, TMessage } from 'librechat-data-provider';
import type { QueuedMessage } from '~/store/families';
+import useQueueDrain from '../useQueueDrain';
import useSteering from '../useSteering';
import store from '~/store';
@@ -260,6 +261,8 @@ describe('useSteering', () => {
stopGenerating: jest.fn(),
}),
queue: useQueue(CONVO_ID),
+ settledReceipts: useRecoilValue(store.settledQueuedTurnReceiptsByConvoId(CONVO_ID)),
+ pendingEnqueueIds: useRecoilValue(store.pendingQueuedTurnEnqueueIdsByConvoId(CONVO_ID)),
}),
{ wrapper },
);
@@ -335,6 +338,377 @@ describe('useSteering', () => {
);
});
+ it('applies an admitted POST replay to the consumed predecessor set', async () => {
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'server-admitted-replay',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ });
+ });
+ const { result } = setupServerQueue(
+ withActiveGeneration(({ set }) => {
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ {
+ id: 'unrelated-server-row',
+ clientRequestId: 'unrelated-request',
+ text: 'keep me',
+ createdAt: 50,
+ server: { id: 'unrelated-server-id', status: 'queued', revision: 1 },
+ },
+ ]);
+ }),
+ );
+
+ await act(async () => {
+ result.current.steering.queueFromComposer('already admitted');
+ await Promise.resolve();
+ });
+
+ await waitFor(() =>
+ expect(result.current.settledReceipts).toEqual([
+ {
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]),
+ );
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({ id: 'unrelated-server-row', text: 'keep me' }),
+ ]);
+ });
+
+ it('keeps an old-server admitted receipt uncertain until its exact boundary is known', async () => {
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'legacy-admitted-replay',
+ status: 'admitted',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ });
+ });
+ const { result } = setupServerQueue();
+
+ await act(async () => {
+ result.current.steering.queueFromComposer('await exact admission boundary');
+ await Promise.resolve();
+ });
+
+ await waitFor(() =>
+ expect(result.current.queue).toEqual([
+ expect.objectContaining({
+ text: 'await exact admission boundary',
+ clientRequestId: expect.stringMatching(UUID_V4_PATTERN),
+ server: expect.objectContaining({
+ id: 'legacy-admitted-replay',
+ status: 'uncertain',
+ }),
+ }),
+ ]),
+ );
+ expect(result.current.settledReceipts).toEqual([]);
+ expect(mockUseAgentQueuedTurns.mock.calls.at(-1)?.[2]).toEqual([
+ result.current.queue[0].clientRequestId,
+ ]);
+ });
+
+ it('does not let a stale queued POST erase incomplete admission evidence', async () => {
+ let releaseQueuedPost = () => undefined;
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ releaseQueuedPost = () =>
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'legacy-admitted-race',
+ status: 'queued',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(100).toISOString(),
+ });
+ });
+ const rendered = setupServerQueue();
+
+ await act(async () => {
+ rendered.result.current.steering.queueFromComposer('hold incomplete admission');
+ await Promise.resolve();
+ });
+ await waitFor(() => expect(mockEnqueueQueuedTurn).toHaveBeenCalledTimes(1));
+ const input = mockEnqueueQueuedTurn.mock.calls[0][0];
+ mockServerQueuedTurns = [
+ {
+ ...input,
+ queuedTurnId: 'legacy-admitted-race',
+ status: 'admitted',
+ revision: 2,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ },
+ ];
+ rendered.rerender();
+
+ await waitFor(() =>
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'legacy-admitted-race',
+ status: 'uncertain',
+ }),
+ );
+ expect(rendered.result.current.settledReceipts).toEqual([
+ { clientRequestId: input.clientRequestId, status: 'admitted_pending_boundary' },
+ ]);
+
+ await act(async () => {
+ releaseQueuedPost();
+ await Promise.resolve();
+ });
+ expect(rendered.result.current.queue).toHaveLength(1);
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'legacy-admitted-race',
+ status: 'uncertain',
+ });
+ expect(rendered.result.current.pendingEnqueueIds).toEqual([]);
+ expect(rendered.result.current.settledReceipts).toEqual([]);
+ });
+
+ it('does not let a stale queued POST response resurrect a terminal receipt', async () => {
+ let releaseQueuedPost = () => undefined;
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ releaseQueuedPost = () =>
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'server-stale-post',
+ status: 'queued',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(100).toISOString(),
+ });
+ });
+ const rendered = setupServerQueue();
+
+ await act(async () => {
+ rendered.result.current.steering.queueFromComposer('settle before POST returns');
+ await Promise.resolve();
+ });
+ await waitFor(() => expect(mockEnqueueQueuedTurn).toHaveBeenCalledTimes(1));
+ const input = mockEnqueueQueuedTurn.mock.calls[0][0];
+ mockServerQueuedTurns = [
+ {
+ ...input,
+ queuedTurnId: 'server-terminal-get',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ revision: 2,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ },
+ ];
+ rendered.rerender();
+
+ await waitFor(() => expect(rendered.result.current.queue).toEqual([]));
+ await act(async () => {
+ releaseQueuedPost();
+ await Promise.resolve();
+ });
+
+ expect(rendered.result.current.queue).toEqual([]);
+ expect(rendered.result.current.settledReceipts).toEqual([
+ {
+ clientRequestId: input.clientRequestId,
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 41,
+ },
+ ]);
+ });
+
+ it('does not let a stale queued POST response resurrect a cancelled row', async () => {
+ let releaseQueuedPost = () => undefined;
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ releaseQueuedPost = () =>
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'server-cancel-race',
+ status: 'queued',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(100).toISOString(),
+ });
+ });
+ const rendered = setupServerQueue();
+
+ await act(async () => {
+ rendered.result.current.steering.queueFromComposer('cancel before POST returns');
+ await Promise.resolve();
+ });
+ await waitFor(() => expect(mockEnqueueQueuedTurn).toHaveBeenCalledTimes(1));
+ const input = mockEnqueueQueuedTurn.mock.calls[0][0];
+ mockServerQueuedTurns = [
+ {
+ ...input,
+ queuedTurnId: 'server-cancel-race',
+ status: 'queued',
+ revision: 2,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ },
+ ];
+ rendered.rerender();
+ await waitFor(() =>
+ expect(rendered.result.current.queue[0].server?.id).toBe('server-cancel-race'),
+ );
+ mockCancelQueuedTurn.mockResolvedValueOnce({
+ ...input,
+ queuedTurnId: 'server-cancel-race',
+ status: 'cancelled',
+ revision: 3,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(300).toISOString(),
+ });
+
+ await act(async () => {
+ await rendered.result.current.steering.discardQueued(rendered.result.current.queue[0]);
+ });
+ expect(rendered.result.current.queue[0].server).toBeUndefined();
+ expect(rendered.result.current.settledReceipts).toEqual([
+ { clientRequestId: input.clientRequestId, status: 'cancelled' },
+ ]);
+
+ await act(async () => {
+ releaseQueuedPost();
+ await Promise.resolve();
+ });
+ expect(rendered.result.current.queue).toEqual([
+ expect.objectContaining({
+ text: 'cancel before POST returns',
+ clientRequestId: input.clientRequestId,
+ }),
+ ]);
+ expect(rendered.result.current.queue[0].server).toBeUndefined();
+ expect(rendered.result.current.pendingEnqueueIds).toEqual([]);
+ expect(rendered.result.current.settledReceipts).toEqual([]);
+ });
+
+ it('does not let a stale queued POST response replace a definitive dead snapshot', async () => {
+ let releaseQueuedPost = () => undefined;
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ releaseQueuedPost = () =>
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'server-dead-race',
+ status: 'queued',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(100).toISOString(),
+ });
+ });
+ const rendered = setupServerQueue();
+
+ await act(async () => {
+ rendered.result.current.steering.queueFromComposer('fail before POST returns');
+ await Promise.resolve();
+ });
+ await waitFor(() => expect(mockEnqueueQueuedTurn).toHaveBeenCalledTimes(1));
+ const input = mockEnqueueQueuedTurn.mock.calls[0][0];
+ mockServerQueuedTurns = [
+ {
+ ...input,
+ queuedTurnId: 'server-dead-race',
+ status: 'dead',
+ revision: 2,
+ failure: { code: 'PREDECESSOR_ABORTED', message: 'The predecessor aborted' },
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ },
+ ];
+ rendered.rerender();
+
+ await waitFor(() =>
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'server-dead-race',
+ status: 'rejected',
+ errorCode: 'PREDECESSOR_ABORTED',
+ }),
+ );
+ expect(rendered.result.current.settledReceipts).toEqual([
+ { clientRequestId: input.clientRequestId, status: 'dead' },
+ ]);
+
+ await act(async () => {
+ releaseQueuedPost();
+ await Promise.resolve();
+ });
+ expect(rendered.result.current.queue).toHaveLength(1);
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'server-dead-race',
+ status: 'rejected',
+ errorCode: 'PREDECESSOR_ABORTED',
+ });
+ expect(rendered.result.current.pendingEnqueueIds).toEqual([]);
+ expect(rendered.result.current.settledReceipts).toEqual([]);
+ });
+
+ it('does not let a stale queued POST replace indeterminate GET evidence', async () => {
+ let releaseQueuedPost = () => undefined;
+ mockEnqueueQueuedTurn.mockImplementation((input, options) => {
+ releaseQueuedPost = () =>
+ options.onSuccess({
+ ...input,
+ queuedTurnId: 'server-indeterminate-race',
+ status: 'queued',
+ revision: 1,
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(100).toISOString(),
+ });
+ });
+ const rendered = setupServerQueue();
+
+ await act(async () => {
+ rendered.result.current.steering.queueFromComposer('unknown before POST returns');
+ await Promise.resolve();
+ });
+ await waitFor(() => expect(mockEnqueueQueuedTurn).toHaveBeenCalledTimes(1));
+ const input = mockEnqueueQueuedTurn.mock.calls[0][0];
+ mockServerQueuedTurns = [
+ {
+ ...input,
+ queuedTurnId: 'server-indeterminate-race',
+ status: 'claimed',
+ revision: 2,
+ failure: { code: 'ADMISSION_INDETERMINATE' },
+ createdAt: new Date(100).toISOString(),
+ updatedAt: new Date(200).toISOString(),
+ },
+ ];
+ rendered.rerender();
+
+ await waitFor(() =>
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'server-indeterminate-race',
+ status: 'indeterminate',
+ revision: 2,
+ }),
+ );
+ expect(rendered.result.current.settledReceipts).toEqual([
+ { clientRequestId: input.clientRequestId, status: 'indeterminate' },
+ ]);
+
+ await act(async () => {
+ releaseQueuedPost();
+ await Promise.resolve();
+ });
+ expect(rendered.result.current.queue[0].server).toMatchObject({
+ id: 'server-indeterminate-race',
+ status: 'indeterminate',
+ revision: 2,
+ });
+ expect(rendered.result.current.pendingEnqueueIds).toEqual([]);
+ expect(rendered.result.current.settledReceipts).toEqual([]);
+ });
+
it('anchors enqueue to the selected branch tail instead of a later hidden sibling', async () => {
mockMessages = [
{
@@ -482,16 +856,144 @@ describe('useSteering', () => {
updatedAt: new Date(200).toISOString(),
},
];
- const { result } = setupServerQueue();
+ const rendered = setupServerQueue();
- await waitFor(() => expect(result.current.queue).toHaveLength(1));
- expect(result.current.queue[0]).toMatchObject({
+ await waitFor(() => expect(rendered.result.current.queue).toHaveLength(1));
+ expect(rendered.result.current.queue[0]).toMatchObject({
id: 'client-snapshot-1',
text: 'restored after reload',
server: { id: 'server-snapshot-1', status: 'queued', revision: 3 },
});
});
+ it('fences the predecessor boundary when the server receipt is admitted', async () => {
+ mockServerQueuedTurns = [
+ {
+ queuedTurnId: 'server-admitted-1',
+ clientRequestId: 'client-admitted-1',
+ conversationId: CONVO_ID,
+ parentMessageId: 'visible-assistant-tail',
+ text: 'already started by the server',
+ status: 'admitted',
+ revision: 4,
+ expectedPredecessorCreatedAt: 41,
+ effectivePredecessorCreatedAt: 42,
+ createdAt: new Date(200).toISOString(),
+ updatedAt: new Date(300).toISOString(),
+ },
+ ];
+ const { result } = setupServerQueue();
+
+ await waitFor(() =>
+ expect(result.current.settledReceipts).toEqual([
+ {
+ clientRequestId: 'client-admitted-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 42,
+ },
+ ]),
+ );
+ expect(result.current.queue).toEqual([]);
+ });
+
+ it('settles an explicit root admission without waiting for a timestamp boundary', async () => {
+ mockServerQueuedTurns = [
+ {
+ queuedTurnId: 'server-root-admitted',
+ clientRequestId: 'client-root-admitted',
+ conversationId: CONVO_ID,
+ parentMessageId: 'visible-assistant-tail',
+ text: 'started from the conversation root',
+ status: 'admitted',
+ revision: 4,
+ rootPredecessor: true,
+ createdAt: new Date(200).toISOString(),
+ updatedAt: new Date(300).toISOString(),
+ },
+ ];
+ const { result } = setupServerQueue();
+
+ await waitFor(() => expect(result.current.queue).toEqual([]));
+ expect(result.current.settledReceipts).toEqual([]);
+ });
+
+ it('keeps the terminal boundary when an old-server admission lacks its exact fence', async () => {
+ mockServerQueuedTurns = [
+ {
+ queuedTurnId: 'server-admitted-race',
+ clientRequestId: 'client-admitted-race',
+ conversationId: CONVO_ID,
+ parentMessageId: 'visible-assistant-tail',
+ text: 'server-owned successor',
+ status: 'admitted',
+ revision: 4,
+ expectedPredecessorCreatedAt: 41,
+ createdAt: new Date(200).toISOString(),
+ updatedAt: new Date(300).toISOString(),
+ },
+ ];
+ const ask = jest.fn();
+ const sendNow = jest.fn();
+ const wrapper = ({ children }: { children: React.ReactNode }) => (
+ <RecoilRoot
+ initializeState={withActiveGeneration(({ set }) => {
+ set(store.isSubmittingFamily(0), false);
+ set(store.runEndByIndex(0), {
+ conversationId: CONVO_ID,
+ outcome: 'completed',
+ endedAt: 200,
+ generationCreatedAt: 41,
+ });
+ set(store.queuedMessagesByConvoId(CONVO_ID), [
+ {
+ id: 'client-admitted-race',
+ clientRequestId: 'client-admitted-race',
+ text: 'server-owned successor',
+ createdAt: 100,
+ server: { id: 'server-admitted-race', status: 'claimed', revision: 3 },
+ },
+ { id: 'local-successor', text: 'must wait', createdAt: 101 },
+ ]);
+ })}
+ >
+ {children}
+ </RecoilRoot>
+ );
+ const { result } = renderHook(
+ () => {
+ useSteering({
+ index: 0,
+ conversationId: CONVO_ID,
+ conversation: agentsConversation,
+ isSubmitting: false,
+ answerModeActive: false,
+ sendNow,
+ stopGenerating: jest.fn(),
+ });
+ useQueueDrain(0, CONVO_ID, ask);
+ return {
+ queue: useQueue(CONVO_ID),
+ runEnd: useRecoilValue(store.runEndByIndex(0)),
+ };
+ },
+ { wrapper },
+ );
+
+ await waitFor(() =>
+ expect(result.current.queue[0]).toMatchObject({
+ id: 'client-admitted-race',
+ server: { id: 'server-admitted-race', status: 'uncertain' },
+ }),
+ );
+ expect(result.current.runEnd).not.toBeNull();
+ expect(result.current.queue.map((item) => item.id)).toEqual([
+ 'client-admitted-race',
+ 'local-successor',
+ ]);
+ expect(ask).not.toHaveBeenCalled();
+ expect(sendNow).not.toHaveBeenCalled();
+ });
+
it('uses server sequence instead of Mongo timestamps for projected queue order', async () => {
mockServerQueuedTurns = [
{
@@ -542,10 +1044,10 @@ describe('useSteering', () => {
},
];
mockCancelQueuedTurn.mockResolvedValueOnce({ status: 'cancelled' });
- const { result } = setupServerQueue();
+ const rendered = setupServerQueue();
- await waitFor(() => expect(result.current.queue).toHaveLength(1));
- expect(result.current.queue[0]).toMatchObject({
+ await waitFor(() => expect(rendered.result.current.queue).toHaveLength(1));
+ expect(rendered.result.current.queue[0]).toMatchObject({
text: 'recover this failed turn',
server: {
id: 'server-dead-1',
@@ -556,15 +1058,48 @@ describe('useSteering', () => {
});
await act(async () => {
- await expect(result.current.steering.discardQueued(result.current.queue[0])).resolves.toBe(
- true,
- );
+ await expect(
+ rendered.result.current.steering.discardQueued(rendered.result.current.queue[0]),
+ ).resolves.toBe(true);
});
expect(mockCancelQueuedTurn).toHaveBeenCalledWith({
conversationId: CONVO_ID,
queuedTurnId: 'server-dead-1',
});
- expect(result.current.queue[0].server).toBeUndefined();
+ expect(rendered.result.current.queue[0].server).toBeUndefined();
+ });
+
+ it('projects an indeterminate claim as non-progressing reconciliation work', async () => {
+ mockServerQueuedTurns = [
+ {
+ queuedTurnId: 'server-indeterminate-1',
+ clientRequestId: 'client-indeterminate-1',
+ conversationId: CONVO_ID,
+ parentMessageId: 'visible-assistant-tail',
+ text: 'external result is unknown',
+ status: 'claimed',
+ revision: 4,
+ failure: {
+ code: 'ADMISSION_INDETERMINATE',
+ message: 'Awaiting exact source evidence',
+ },
+ createdAt: new Date(250).toISOString(),
+ updatedAt: new Date(300).toISOString(),
+ },
+ ];
+ const rendered = setupServerQueue();
+
+ await waitFor(() => expect(rendered.result.current.queue).toHaveLength(1));
+ expect(rendered.result.current.queue[0]).toMatchObject({
+ text: 'external result is unknown',
+ server: {
+ id: 'server-indeterminate-1',
+ status: 'indeterminate',
+ errorCode: 'ADMISSION_INDETERMINATE',
+ errorMessage: 'Awaiting exact source evidence',
+ },
+ });
+ expect(rendered.result.current.settledReceipts).toEqual([]);
});
it('cancels the durable copy before downgrading a row to local control', async () => {
diff --git a/client/src/hooks/Chat/useQueueDrain.ts b/client/src/hooks/Chat/useQueueDrain.ts
index e636c113a2..7039adc21d 100644
--- a/client/src/hooks/Chat/useQueueDrain.ts
+++ b/client/src/hooks/Chat/useQueueDrain.ts
@@ -89,6 +89,13 @@ export default function useQueueDrain(
* during the first turn stay keyed here until that run ends. Renewing only
* the active id would skip them for the whole of that run. */
const newConvoQueue = useRecoilValue(store.queuedMessagesByConvoId(Constants.NEW_CONVO));
+ /** Receipt settlement can consume the parked terminal boundary without
+ * changing whether another server-owned row remains. Subscribe here so the
+ * drain effect observes that durable transition instead of reading it only
+ * through a callback snapshot whose other dependencies stayed unchanged. */
+ const settledQueuedTurnReceipts = useRecoilValue(
+ store.settledQueuedTurnReceiptsByConvoId(activeConversationId ?? Constants.NEW_CONVO),
+ );
const hasServerOwnedQueue = [...ownQueue, ...newConvoQueue].some((item) => item.server != null);
/* Deduped because the two subscriptions are the same atom before migration.
@@ -238,6 +245,57 @@ export default function useQueueDrain(
: ownQueue;
const shouldDrain = end.outcome === 'completed' || interruptArmed;
+ const settledReceipts = snapshot
+ .getLoadable(store.settledQueuedTurnReceiptsByConvoId(conversationId))
+ .getValue();
+ const pendingEnqueueIds = snapshot
+ .getLoadable(store.pendingQueuedTurnEnqueueIdsByConvoId(conversationId))
+ .getValue();
+ const consumedReceiptIndex = settledReceipts.findIndex(
+ (receipt) =>
+ receipt.status === 'admitted' &&
+ receipt.boundaryConsumed !== true &&
+ end.generationCreatedAt != null &&
+ receipt.effectivePredecessorCreatedAt === end.generationCreatedAt,
+ );
+ const consumedByServerAdmission = consumedReceiptIndex >= 0;
+ const consumeEnd = () => {
+ if (fromParked && activeConversationId) {
+ set(store.pendingRunEndByConvoId(activeConversationId), null);
+ } else {
+ set(store.runEndByIndex(index), null);
+ }
+ if (matchingIndexArm) {
+ set(store.drainAfterAbortByIndex(index), false);
+ }
+ };
+ if (consumedByServerAdmission) {
+ /** Admission consumed this terminal boundary on the server. A late
+ * client terminal observation cannot authorize a second successor. */
+ if (shouldMigrate && newConvoQueue.length > 0) {
+ set(store.queuedMessagesByConvoId(Constants.NEW_CONVO), []);
+ set(store.queuedMessagesByConvoId(conversationId), merged);
+ }
+ set(store.settledQueuedTurnReceiptsByConvoId(conversationId), (previous) => {
+ let consumed = false;
+ return previous.flatMap((receipt) => {
+ if (
+ !consumed &&
+ receipt.status === 'admitted' &&
+ receipt.boundaryConsumed !== true &&
+ receipt.effectivePredecessorCreatedAt === end.generationCreatedAt
+ ) {
+ consumed = true;
+ return pendingEnqueueIds.includes(receipt.clientRequestId)
+ ? [{ ...receipt, boundaryConsumed: true }]
+ : [];
+ }
+ return [receipt];
+ });
+ });
+ consumeEnd();
+ return null;
+ }
/** A server-owned Agent row means the backend owns the next fresh-turn
* admission. Do not let a legacy/recovered local row race or overtake
* it. Keep this terminal boundary available until the authoritative
@@ -253,14 +311,7 @@ export default function useQueueDrain(
// Consume only after server authority has yielded the boundary — a
// hard double-fire guard even if the effect re-runs before propagation.
- if (fromParked && activeConversationId) {
- set(store.pendingRunEndByConvoId(activeConversationId), null);
- } else {
- set(store.runEndByIndex(index), null);
- }
- if (matchingIndexArm) {
- set(store.drainAfterAbortByIndex(index), false);
- }
+ consumeEnd();
const next = shouldDrain ? (merged[0] ?? null) : null;
const remainder = next ? merged.slice(1) : merged;
@@ -362,6 +413,7 @@ export default function useQueueDrain(
restoreQueued,
markFilesUsage,
hasServerOwnedQueue,
+ settledQueuedTurnReceipts,
ask,
]);
}
diff --git a/client/src/hooks/Chat/useSteering.ts b/client/src/hooks/Chat/useSteering.ts
index 626a5c8114..d150af96d4 100644
--- a/client/src/hooks/Chat/useSteering.ts
+++ b/client/src/hooks/Chat/useSteering.ts
@@ -14,7 +14,13 @@ import type {
TConversation,
TMessageContentParts,
} from 'librechat-data-provider';
-import type { RunEnd, PendingSteer, QueuedMessage, QueuedMessageOrigin } from '~/store/families';
+import type {
+ RunEnd,
+ PendingSteer,
+ QueuedMessage,
+ QueuedMessageOrigin,
+ SettledQueuedTurnReceipt,
+} from '~/store/families';
import type { AgentQueuedTurnReceipt, GenerationProtocolVersion } from '~/data-provider';
import type { ExtendedFile, FileSetter } from '~/common';
import {
@@ -172,6 +178,76 @@ function queuedTurnCreatedAt(receipt: AgentQueuedTurnReceipt): number {
return Number.isFinite(createdAt) ? createdAt : 0;
}
+type QueuedTurnReceiptSource = 'snapshot' | 'enqueue' | 'direct';
+
+function isAdmissionIndeterminateReceipt(receipt: AgentQueuedTurnReceipt): boolean {
+ return (
+ receipt.failure?.code === 'ADMISSION_INDETERMINATE' &&
+ (receipt.status === 'claimed' || receipt.status === 'dead')
+ );
+}
+
+function settledEvidenceForReceipt(
+ receipt: AgentQueuedTurnReceipt,
+): SettledQueuedTurnReceipt | undefined {
+ if (
+ receipt.status === 'admitted' &&
+ (receipt.effectivePredecessorCreatedAt != null || receipt.rootPredecessor === true)
+ ) {
+ return {
+ clientRequestId: receipt.clientRequestId,
+ status: 'admitted',
+ ...(receipt.effectivePredecessorCreatedAt != null && {
+ effectivePredecessorCreatedAt: receipt.effectivePredecessorCreatedAt,
+ }),
+ ...(receipt.rootPredecessor === true && { rootPredecessor: true }),
+ };
+ }
+ if (receipt.status === 'admitted') {
+ return {
+ clientRequestId: receipt.clientRequestId,
+ status: 'admitted_pending_boundary',
+ };
+ }
+ if (isAdmissionIndeterminateReceipt(receipt)) {
+ return { clientRequestId: receipt.clientRequestId, status: 'indeterminate' };
+ }
+ if (receipt.status === 'cancelled') {
+ return { clientRequestId: receipt.clientRequestId, status: 'cancelled' };
+ }
+ if (receipt.status === 'dead') {
+ return { clientRequestId: receipt.clientRequestId, status: 'dead' };
+ }
+ return undefined;
+}
+
+/** Client receipt knowledge is a monotonic evidence lattice. Enqueue is the
+ * only source that can carry a pre-scheduling snapshot, so it never weakens
+ * evidence already observed by GET/direct settlement. Indeterminate evidence
+ * can advance only through a later authoritative terminal observation. */
+function mergeSettledQueuedTurnEvidence(
+ existing: SettledQueuedTurnReceipt | undefined,
+ receipt: AgentQueuedTurnReceipt,
+ source: QueuedTurnReceiptSource,
+): SettledQueuedTurnReceipt | undefined {
+ const incoming = settledEvidenceForReceipt(receipt);
+ if (existing == null) {
+ return incoming;
+ }
+ if (existing.status === 'admitted_pending_boundary' && incoming?.status === 'admitted') {
+ return incoming;
+ }
+ if (
+ existing.status === 'indeterminate' &&
+ source !== 'enqueue' &&
+ incoming != null &&
+ incoming.status !== 'indeterminate'
+ ) {
+ return incoming;
+ }
+ return existing;
+}
+
function toQueuedTurnFileRefs(files: TMessage['files']): TAgentQueuedTurnFileRef[] | undefined {
const refs = (files ?? []).flatMap((file): TAgentQueuedTurnFileRef[] => {
if (typeof file.file_id !== 'string' || file.file_id.length === 0) {
@@ -195,6 +271,8 @@ function toQueuedTurnFileRefs(files: TMessage['files']): TAgentQueuedTurnFileRef
function reconcileServerQueuedTurns(
previous: QueuedMessage[],
receipts: AgentQueuedTurnReceipt[],
+ settledByRequestId: ReadonlyMap<string, SettledQueuedTurnReceipt>,
+ authoritativeSnapshot = true,
): QueuedMessage[] {
const previousByClientRequestId = new Map(
previous.flatMap((item) =>
@@ -203,10 +281,30 @@ function reconcileServerQueuedTurns(
);
const observedClientRequestIds = new Set(receipts.map((receipt) => receipt.clientRequestId));
const projected = receipts.flatMap((receipt): QueuedMessage[] => {
- if (receipt.status === 'admitted' || receipt.status === 'cancelled') {
+ const settled = settledByRequestId.get(receipt.clientRequestId);
+ const admissionIndeterminate = isAdmissionIndeterminateReceipt(receipt);
+ if (settled?.status === 'admitted' || settled?.status === 'cancelled') {
return [];
}
const optimistic = previousByClientRequestId.get(receipt.clientRequestId);
+ if (
+ (settled?.status === 'dead' ||
+ settled?.status === 'admitted_pending_boundary' ||
+ (settled?.status === 'indeterminate' && !admissionIndeterminate)) &&
+ receipt.status !== 'dead' &&
+ receipt.status !== 'admitted'
+ ) {
+ return optimistic == null ? [] : [optimistic];
+ }
+ const boundaryPending = receipt.status === 'admitted';
+ let status: NonNullable<QueuedMessage['server']>['status'] = 'rejected';
+ if (admissionIndeterminate) {
+ status = 'indeterminate';
+ } else if (boundaryPending) {
+ status = 'uncertain';
+ } else if (receipt.status === 'queued' || receipt.status === 'claimed') {
+ status = receipt.status;
+ }
return [
{
id: optimistic?.id ?? receipt.clientRequestId,
@@ -226,8 +324,11 @@ function reconcileServerQueuedTurns(
...(receipt.priority === true && { priority: true }),
server: {
id: receipt.queuedTurnId,
- status: receipt.status === 'dead' ? 'rejected' : receipt.status,
+ status,
revision: receipt.revision,
+ ...(boundaryPending && {
+ uncertainSince: optimistic?.server?.uncertainSince ?? Date.now(),
+ }),
...(receipt.position != null && { position: receipt.position }),
...(receipt.failure?.code != null && { errorCode: receipt.failure.code }),
...(receipt.failure?.message != null && { errorMessage: receipt.failure.message }),
@@ -239,12 +340,24 @@ function reconcileServerQueuedTurns(
if (item.server == null) {
return true;
}
+ const settled =
+ item.clientRequestId == null ? undefined : settledByRequestId.get(item.clientRequestId);
+ if (settled?.status === 'admitted' || settled?.status === 'cancelled') {
+ return false;
+ }
+ if (settled?.status === 'dead' || settled?.status === 'admitted_pending_boundary') {
+ return item.clientRequestId == null || !observedClientRequestIds.has(item.clientRequestId);
+ }
if (item.clientRequestId == null || observedClientRequestIds.has(item.clientRequestId)) {
return false;
}
+ if (!authoritativeSnapshot) {
+ return true;
+ }
return (
item.server.status === 'sending' ||
item.server.status === 'uncertain' ||
+ item.server.status === 'indeterminate' ||
item.server.status === 'rejected'
);
});
@@ -363,12 +476,93 @@ export default function useSteering({
store.activeGenerationProtocolVersionByConvoId(queueKey),
);
+ const applyQueuedTurnReceipts = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (receipts: AgentQueuedTurnReceipt[], source: QueuedTurnReceiptSource = 'snapshot') => {
+ const previousSettled = snapshot
+ .getLoadable(store.settledQueuedTurnReceiptsByConvoId(queueKey))
+ .getValue();
+ const previousPending = snapshot
+ .getLoadable(store.pendingQueuedTurnEnqueueIdsByConvoId(queueKey))
+ .getValue();
+ const completedRequestIds = new Set(
+ source === 'enqueue' ? receipts.map((receipt) => receipt.clientRequestId) : [],
+ );
+ const nextPending = previousPending.filter((id) => !completedRequestIds.has(id));
+ const settledByRequestId = new Map(
+ previousSettled.map((receipt) => [receipt.clientRequestId, receipt]),
+ );
+ for (const receipt of receipts) {
+ const existing = settledByRequestId.get(receipt.clientRequestId);
+ const settled = mergeSettledQueuedTurnEvidence(existing, receipt, source);
+ if (settled == null) {
+ continue;
+ }
+ settledByRequestId.set(receipt.clientRequestId, settled);
+ }
+ const terminalForReconciliation = new Map(settledByRequestId);
+ const pendingRequestIds = new Set(nextPending);
+ const nextSettled = [...settledByRequestId.values()].filter(
+ (receipt) =>
+ (receipt.status === 'admitted' &&
+ receipt.rootPredecessor !== true &&
+ receipt.boundaryConsumed !== true) ||
+ pendingRequestIds.has(receipt.clientRequestId),
+ );
+ if (source === 'enqueue') {
+ set(store.pendingQueuedTurnEnqueueIdsByConvoId(queueKey), nextPending);
+ }
+ if (
+ nextSettled.length !== previousSettled.length ||
+ nextSettled.some((receipt, index) => receipt !== previousSettled[index])
+ ) {
+ set(store.settledQueuedTurnReceiptsByConvoId(queueKey), nextSettled);
+ }
+ if (source !== 'direct') {
+ set(store.queuedMessagesByConvoId(queueKey), (previous) =>
+ reconcileServerQueuedTurns(
+ previous,
+ receipts,
+ terminalForReconciliation,
+ source === 'snapshot',
+ ),
+ );
+ }
+ },
+ [queueKey],
+ );
+
+ const finishQueuedTurnEnqueue = useRecoilCallback(
+ ({ snapshot, set }) =>
+ (clientRequestId: string): boolean => {
+ const settledReceipts = snapshot
+ .getLoadable(store.settledQueuedTurnReceiptsByConvoId(queueKey))
+ .getValue();
+ const settled = settledReceipts.find(
+ (receipt) => receipt.clientRequestId === clientRequestId,
+ );
+ set(store.pendingQueuedTurnEnqueueIdsByConvoId(queueKey), (previous) =>
+ previous.filter((id) => id !== clientRequestId),
+ );
+ if (
+ settled != null &&
+ (settled.status !== 'admitted' || settled.boundaryConsumed === true)
+ ) {
+ set(store.settledQueuedTurnReceiptsByConvoId(queueKey), (previous) =>
+ previous.filter((receipt) => receipt.clientRequestId !== clientRequestId),
+ );
+ }
+ return settled != null;
+ },
+ [queueKey],
+ );
+
useEffect(() => {
if (!serverQueueEnabled || serverQueuedTurns == null) {
return;
}
- setQueuedMessages((previous) => reconcileServerQueuedTurns(previous, serverQueuedTurns));
- }, [serverQueueEnabled, serverQueuedTurns, setQueuedMessages]);
+ applyQueuedTurnReceipts(serverQueuedTurns);
+ }, [serverQueueEnabled, serverQueuedTurns, applyQueuedTurnReceipts]);
useEffect(() => {
if (!serverQueueEnabled || nextReconciliationExpiry == null) {
return;
@@ -796,6 +990,9 @@ export default function useSteering({
if (!serverOwned || clientRequestId == null) {
return;
}
+ set(store.pendingQueuedTurnEnqueueIdsByConvoId(queueKey), (previous) =>
+ previous.includes(clientRequestId) ? previous : [...previous, clientRequestId],
+ );
const serverFiles = toQueuedTurnFileRefs(item.files);
/** Commit the optimistic Recoil row before mutation callbacks can
* reconcile it. This also makes synchronous test/adaptor completions
@@ -820,29 +1017,12 @@ export default function useSteering({
},
{
onSuccess: (receipt) => {
- updateQueuedMessage(item.id, (current) => {
- if (receipt.status === 'admitted' || receipt.status === 'cancelled') {
- return null;
- }
- return {
- ...current,
- text: receipt.text,
- createdAt: queuedTurnCreatedAt(receipt),
- parentMessageId: receipt.parentMessageId,
- server: {
- id: receipt.queuedTurnId,
- status: receipt.status === 'dead' ? 'rejected' : receipt.status,
- revision: receipt.revision,
- ...(receipt.position != null && { position: receipt.position }),
- ...(receipt.failure?.code != null && { errorCode: receipt.failure.code }),
- ...(receipt.failure?.message != null && {
- errorMessage: receipt.failure.message,
- }),
- },
- };
- });
+ applyQueuedTurnReceipts([receipt], 'enqueue');
},
onError: (error) => {
+ if (finishQueuedTurnEnqueue(clientRequestId)) {
+ return;
+ }
updateQueuedMessage(item.id, (current) => {
if (isDefiniteQueuedTurnsUnsupported(error)) {
const {
@@ -891,6 +1071,8 @@ export default function useSteering({
markQueuedFilesUsage,
activeGenerationCreatedAt,
enqueueAgentQueuedTurn,
+ applyQueuedTurnReceipts,
+ finishQueuedTurnEnqueue,
updateQueuedMessage,
],
);
@@ -1115,6 +1297,7 @@ export default function useSteering({
});
return false;
}
+ applyQueuedTurnReceipts([receipt], 'direct');
return downgradeServerQueuedTurn(item.id);
} catch {
showToast({
@@ -1159,6 +1342,7 @@ export default function useSteering({
[
cancelSteer,
cancelAgentQueuedTurn,
+ applyQueuedTurnReceipts,
conversationId,
downgradeQueuedRecovery,
downgradeServerQueuedTurn,
diff --git a/client/src/locales/en/translation.json b/client/src/locales/en/translation.json
index 7e27e2ad98..fa36e7c78e 100644
--- a/client/src/locales/en/translation.json
+++ b/client/src/locales/en/translation.json
@@ -1791,6 +1791,7 @@
"com_ui_queued_messages": "Queued messages",
"com_ui_queued_quote_count": "{{0}} quoted excerpts included with this message",
"com_ui_queued_turn_failed": "Queued message failed",
+ "com_ui_queued_turn_reconciliation_required": "Awaiting reconciliation",
"com_ui_quote_selections": "{{0}} selections",
"com_ui_quotes_queued": "Quotes added for your next message",
"com_ui_ran_n_agents": "Ran {{0}} agents",
diff --git a/client/src/store/families.ts b/client/src/store/families.ts
index 3ae2cb9294..26502fb2d2 100644
--- a/client/src/store/families.ts
+++ b/client/src/store/families.ts
@@ -402,7 +402,7 @@ export type QueuedMessage = {
* after an ambiguous POST could submit the same words twice. */
server?: {
id?: string;
- status: 'sending' | 'uncertain' | 'rejected' | 'queued' | 'claimed';
+ status: 'sending' | 'uncertain' | 'indeterminate' | 'rejected' | 'queued' | 'claimed';
errorCode?: string;
errorMessage?: string;
/** Observation time for a transport-ambiguous enqueue. The logical item
@@ -459,6 +459,29 @@ const queuedMessagesByConvoId = atomFamily<QueuedMessage[], string>({
default: [],
});
+export type SettledQueuedTurnReceipt = {
+ clientRequestId: string;
+ status: 'admitted' | 'admitted_pending_boundary' | 'indeterminate' | 'cancelled' | 'dead';
+ effectivePredecessorCreatedAt?: number;
+ rootPredecessor?: true;
+ boundaryConsumed?: boolean;
+};
+
+/** Monotonic client knowledge of terminal server queue receipts. Admission
+ * records preserve boundary multiplicity by request identity. Other terminal
+ * records exist only while their original enqueue callback is outstanding. */
+const settledQueuedTurnReceiptsByConvoId = atomFamily<SettledQueuedTurnReceipt[], string>({
+ key: 'settledQueuedTurnReceiptsByConvoId',
+ default: [],
+});
+
+/** Enqueue callbacks that can still race newer GET/cancellation evidence.
+ * Entries retire as soon as that one callback settles. */
+const pendingQueuedTurnEnqueueIdsByConvoId = atomFamily<string[], string>({
+ key: 'pendingQueuedTurnEnqueueIdsByConvoId',
+ default: [],
+});
+
/**
* One-shot run-termination signal written by the SSE final/error handlers and
* consumed (reset to null) by `useQueueDrain`. Keyed by chat index like
@@ -788,6 +811,8 @@ export default {
pendingQuotesByConvoId,
pendingSteersByConvoId,
queuedMessagesByConvoId,
+ settledQueuedTurnReceiptsByConvoId,
+ pendingQueuedTurnEnqueueIdsByConvoId,
runEndByIndex,
pendingRunEndByConvoId,
drainAfterAbortByIndex,
diff --git a/packages/api/src/agents/queuedTurnHttp.spec.ts b/packages/api/src/agents/queuedTurnHttp.spec.ts
index 83dc7892d8..559a569e4f 100644
--- a/packages/api/src/agents/queuedTurnHttp.spec.ts
+++ b/packages/api/src/agents/queuedTurnHttp.spec.ts
@@ -67,10 +67,21 @@ describe('Agent queued-turn HTTP admission receipts', () => {
const enqueueAgentQueuedTurn = jest
.fn()
.mockResolvedValueOnce({ turn: turn('queued'), replayed: false });
+ const admitted = {
+ ...turn('admitted'),
+ terminalReceipt: {
+ outcome: 'admitted' as const,
+ settledAt: new Date('2026-08-30T12:01:00Z'),
+ admissionId: 'client-request-1',
+ generationId: 'generation-1',
+ generationCreatedAt: 43,
+ effectivePredecessorCreatedAt: 42,
+ },
+ };
const getAgentQueuedTurnByClientRequestId = jest
.fn()
.mockResolvedValueOnce(null)
- .mockResolvedValueOnce(turn('admitted'));
+ .mockResolvedValueOnce(admitted);
const schedule = jest.fn().mockRejectedValueOnce(new Error('scheduler unavailable'));
const methods = {
getConvo: jest.fn(async () => ({
@@ -104,6 +115,7 @@ describe('Agent queued-turn HTTP admission receipts', () => {
queuedTurnId: 'queued-turn-1',
clientRequestId: 'client-request-1',
status: 'admitted',
+ effectivePredecessorCreatedAt: 42,
},
},
});
@@ -172,6 +184,47 @@ describe('Agent queued-turn HTTP admission receipts', () => {
expect(cancel).toHaveBeenCalledWith(expect.objectContaining({ queuedTurnId: 'queued-turn-1' }));
});
+ it('projects an explicit root admission without a timestamp boundary', async () => {
+ const admitted = {
+ ...turn('admitted'),
+ terminalReceipt: {
+ outcome: 'admitted' as const,
+ settledAt: new Date('2026-08-30T12:01:00Z'),
+ admissionId: 'client-request-1',
+ generationId: 'generation-root',
+ generationCreatedAt: 43,
+ lineagePredecessorId: 'root:message-identity',
+ rootPredecessor: true as const,
+ },
+ };
+ const methods = {
+ getConvo: jest.fn(async () => ({ agent_id: 'agent_1', endpoint: 'agents' })),
+ listAgentQueuedTurnReceipts: jest.fn(async () => [admitted]),
+ };
+ const deps = {
+ methods: methods as unknown as AgentQueuedTurnMethods & {
+ getConvo: typeof methods.getConvo;
+ },
+ lifecycle: { schedule: jest.fn(), cancel: jest.fn() },
+ checkAgentAccess: jest.fn(async () => true),
+ } satisfies AgentQueuedTurnHttpDeps;
+
+ await expect(
+ handleAgentQueuedTurnList({ id: USER_ID }, 'conversation-1', deps),
+ ).resolves.toMatchObject({
+ status: 200,
+ body: {
+ queuedTurns: [
+ {
+ queuedTurnId: 'queued-turn-1',
+ status: 'admitted',
+ rootPredecessor: true,
+ },
+ ],
+ },
+ });
+ });
+
it('retires a cancelled source after its published delivery receipt expires', async () => {
const cancelled = {
...turn('cancelled'),
diff --git a/packages/api/src/agents/queuedTurnHttp.ts b/packages/api/src/agents/queuedTurnHttp.ts
index 765a2b835a..704083ff3e 100644
--- a/packages/api/src/agents/queuedTurnHttp.ts
+++ b/packages/api/src/agents/queuedTurnHttp.ts
@@ -90,6 +90,12 @@ function receipt(
...(turn.expectedPredecessorCreatedAt != null && {
expectedPredecessorCreatedAt: turn.expectedPredecessorCreatedAt,
}),
+ ...(turn.terminalReceipt?.outcome === 'admitted' &&
+ turn.terminalReceipt.effectivePredecessorCreatedAt != null && {
+ effectivePredecessorCreatedAt: turn.terminalReceipt.effectivePredecessorCreatedAt,
+ }),
+ ...(turn.terminalReceipt?.outcome === 'admitted' &&
+ turn.terminalReceipt.rootPredecessor === true && { rootPredecessor: true as const }),
status: turn.status,
...(position != null && { position }),
revision: turn.sequence,
diff --git a/packages/api/src/agents/queuedTurns.spec.ts b/packages/api/src/agents/queuedTurns.spec.ts
index 3757982b30..ca5eb6f100 100644
--- a/packages/api/src/agents/queuedTurns.spec.ts
+++ b/packages/api/src/agents/queuedTurns.spec.ts
@@ -51,6 +51,7 @@ function claim(): AgentQueuedTurnClaim {
clientRequestId: 'client-1',
fingerprint: 'fingerprint-1',
sequence: 1,
+ admissionSlot: true,
status: 'claimed',
priority: false,
text: 'queued words',
@@ -107,7 +108,7 @@ function resolverMethods() {
getEffectiveAgentQueuedTurnPredecessor: jest.fn(
async (
..._args: Parameters<AgentQueuedTurnMethods['getEffectiveAgentQueuedTurnPredecessor']>
- ): Promise<number | undefined> => undefined,
+ ) => null,
),
markAgentQueuedTurnAdmitted: jest.fn(async () => ({
outcome: 'admitted' as const,
@@ -306,6 +307,7 @@ describe('Agent queued-turn continuation', () => {
sourceId: 'queued-turn-1',
claimId: 'trigger-1',
claimBy: 'worker-1',
+ effectivePredecessorCreatedAt: NOW,
},
{
userId: USER_ID,
@@ -325,6 +327,7 @@ describe('Agent queued-turn continuation', () => {
admissionId: 'trigger-1',
generationId: 'conversation-1',
generationCreatedAt: NOW + 1,
+ effectivePredecessorCreatedAt: NOW,
}),
);
});
@@ -342,6 +345,7 @@ describe('Agent queued-turn continuation', () => {
sourceId: 'queued-turn-1',
claimId: 'trigger-1',
claimBy: 'worker-1',
+ effectivePredecessorCreatedAt: NOW,
};
const admission = {
userId: USER_ID,
@@ -361,6 +365,7 @@ describe('Agent queued-turn continuation', () => {
admissionId: 'trigger-1',
generationId: 'conversation-1',
generationCreatedAt: NOW + 1,
+ effectivePredecessorCreatedAt: NOW,
});
spies.hasAgentQueuedTurnAdmissionReceipt.mockResolvedValueOnce(false);
@@ -451,7 +456,10 @@ describe('Agent queued-turn continuation', () => {
const claimed = claim();
claimed.expectedPredecessorCreatedAt = NOW;
spies.claimNextAgentQueuedTurn.mockResolvedValueOnce({ outcome: 'acquired', claim: claimed });
- spies.getEffectiveAgentQueuedTurnPredecessor.mockResolvedValueOnce(NOW + 250);
+ spies.beginAgentQueuedTurnAdmission.mockResolvedValueOnce({
+ outcome: 'started',
+ turn: { ...claimed, admissionEffectivePredecessorCreatedAt: NOW + 250 },
+ });
const resolve = createAgentQueuedTurnResolver({
methods,
getGenerationJob: async () => null,
@@ -459,17 +467,53 @@ describe('Agent queued-turn continuation', () => {
claimBy: 'worker-1',
});
- await expect(resolve(envelope(), { idempotencyKey: 'trigger-1' })).resolves.toMatchObject({
+ const prepared = await resolve(envelope(), { idempotencyKey: 'trigger-1' });
+ expect(prepared).toMatchObject({
status: 'ready',
expectedPredecessorCreatedAt: NOW + 250,
+ admissionSource: { effectivePredecessorCreatedAt: NOW + 250 },
});
- expect(spies.getEffectiveAgentQueuedTurnPredecessor).toHaveBeenCalledWith({
- user: new Types.ObjectId(USER_ID),
- tenantId: 'tenant-1',
+ expect(spies.beginAgentQueuedTurnAdmission).toHaveBeenCalledTimes(1);
+ if (prepared?.status !== 'ready') {
+ throw new Error('Expected a ready queued turn');
+ }
+ await prepared.settleOnAdmission?.({
+ mode: 'continue',
+ status: 'started',
conversationId: 'conversation-1',
- sequence: 1,
- expectedPredecessorCreatedAt: NOW,
+ streamId: 'stream-2',
+ generationCreatedAt: NOW + 500,
+ });
+ expect(spies.markAgentQueuedTurnAdmitted).toHaveBeenCalledWith(
+ expect.objectContaining({
+ queuedTurnId: 'queued-turn-1',
+ effectivePredecessorCreatedAt: NOW + 250,
+ }),
+ );
+ });
+
+ it('dead-letters a legacy admission order that cannot be reconstructed safely', async () => {
+ const { methods, spies } = resolverMethods();
+ (
+ spies.beginAgentQueuedTurnAdmission as unknown as jest.MockedFunction<
+ AgentQueuedTurnMethods['beginAgentQueuedTurnAdmission']
+ >
+ ).mockResolvedValueOnce({
+ outcome: 'order_unavailable',
+ turn: { ...claim(), status: 'dead' },
});
+ const resolve = createAgentQueuedTurnResolver({
+ methods,
+ getGenerationJob: async () => null,
+ now: () => NOW,
+ claimBy: 'worker-1',
+ });
+
+ await expect(resolve(envelope(), { idempotencyKey: 'trigger-1' })).resolves.toEqual({
+ status: 'settled',
+ });
+ expect(spies.releaseAgentQueuedTurn).not.toHaveBeenCalled();
+ expect(spies.beginAgentQueuedTurnAdmission).toHaveBeenCalledTimes(1);
});
it.each([
@@ -776,10 +820,10 @@ describe('Agent queued-turn delivery scheduling', () => {
);
});
- it('reconciles quarantined admission receipts during durable recovery', async () => {
+ it('reconciles a claimed indeterminate legacy admission during durable recovery', async () => {
const turn = {
...queuedTurn('queued-turn-indeterminate', 1),
- status: 'dead' as const,
+ status: 'claimed' as const,
deliveryKey: 'delivery-indeterminate',
deliveryState: 'published' as const,
admissionId: 'delivery-indeterminate',
@@ -841,7 +885,7 @@ describe('Agent queued-turn delivery scheduling', () => {
it('rotates source-fenced ambiguity without trusting transient job-store evidence', async () => {
const turn = {
...queuedTurn('queued-turn-source-fenced', 1),
- status: 'dead' as const,
+ status: 'claimed' as const,
deliveryKey: 'delivery-source-fenced',
deliveryState: 'published' as const,
admissionId: 'delivery-source-fenced',
diff --git a/packages/api/src/agents/queuedTurns.ts b/packages/api/src/agents/queuedTurns.ts
index 4a9fafa26a..6eeb6864f9 100644
--- a/packages/api/src/agents/queuedTurns.ts
+++ b/packages/api/src/agents/queuedTurns.ts
@@ -153,6 +153,9 @@ function parseQueuedTurnAdmissionSource(raw: unknown): AgentContinuationAdmissio
const sourceId = 'sourceId' in raw ? raw.sourceId : undefined;
const claimId = 'claimId' in raw ? raw.claimId : undefined;
const claimBy = 'claimBy' in raw ? raw.claimBy : undefined;
+ const effectivePredecessorCreatedAt =
+ 'effectivePredecessorCreatedAt' in raw ? raw.effectivePredecessorCreatedAt : undefined;
+ const lineagePredecessorId = 'lineagePredecessorId' in raw ? raw.lineagePredecessorId : undefined;
if (
typeof sourceId !== 'string' ||
sourceId.length === 0 ||
@@ -162,11 +165,28 @@ function parseQueuedTurnAdmissionSource(raw: unknown): AgentContinuationAdmissio
claimId.length > 128 ||
typeof claimBy !== 'string' ||
claimBy.length === 0 ||
- claimBy.length > 256
+ claimBy.length > 256 ||
+ (effectivePredecessorCreatedAt != null &&
+ (typeof effectivePredecessorCreatedAt !== 'number' ||
+ !Number.isSafeInteger(effectivePredecessorCreatedAt) ||
+ effectivePredecessorCreatedAt < 0)) ||
+ (lineagePredecessorId != null &&
+ (typeof lineagePredecessorId !== 'string' ||
+ lineagePredecessorId.length === 0 ||
+ lineagePredecessorId.length > 128))
) {
throw new TypeError('Agent queued turn admission source is invalid');
}
- return { source: AGENT_QUEUED_TURN_SOURCE, sourceId, claimId, claimBy };
+ return {
+ source: AGENT_QUEUED_TURN_SOURCE,
+ sourceId,
+ claimId,
+ claimBy,
+ ...(typeof effectivePredecessorCreatedAt === 'number' && {
+ effectivePredecessorCreatedAt,
+ }),
+ ...(typeof lineagePredecessorId === 'string' && { lineagePredecessorId }),
+ };
}
/** Commits the source-owned admission receipt only after the controller has
@@ -195,6 +215,12 @@ async function settleAgentQueuedTurnExecutionAdmission(
admissionMode: 'ordinary',
generationId: input.generationId,
generationCreatedAt: input.generationCreatedAt,
+ ...(source.effectivePredecessorCreatedAt != null && {
+ effectivePredecessorCreatedAt: source.effectivePredecessorCreatedAt,
+ }),
+ ...(source.lineagePredecessorId != null && {
+ lineagePredecessorId: source.lineagePredecessorId,
+ }),
settledAt: new Date(),
});
if (settled.outcome === 'conflict') {
@@ -226,6 +252,12 @@ async function verifyAgentQueuedTurnExecutionAdmission(
admissionId: input.clientRequestId,
generationId: input.generationId,
generationCreatedAt: input.generationCreatedAt,
+ ...(source.effectivePredecessorCreatedAt != null && {
+ effectivePredecessorCreatedAt: source.effectivePredecessorCreatedAt,
+ }),
+ ...(source.lineagePredecessorId != null && {
+ lineagePredecessorId: source.lineagePredecessorId,
+ }),
});
if (!confirmed) {
throw new Error('The queued turn execution admission is not yet confirmed');
@@ -578,17 +610,6 @@ function createAgentQueuedTurnResolver({
return { status: 'settled' };
}
- const effectivePredecessorCreatedAt =
- (await methods.getEffectiveAgentQueuedTurnPredecessor({
- user: claim.user,
- ...(claim.tenantId != null && { tenantId: claim.tenantId }),
- conversationId: claim.conversationId,
- sequence: claim.sequence,
- ...(claim.expectedPredecessorCreatedAt != null && {
- expectedPredecessorCreatedAt: claim.expectedPredecessorCreatedAt,
- }),
- })) ?? claim.expectedPredecessorCreatedAt;
-
const admission = await methods.beginAgentQueuedTurnAdmission({
user: claim.user,
...(claim.tenantId != null && { tenantId: claim.tenantId }),
@@ -607,9 +628,11 @@ function createAgentQueuedTurnResolver({
deferWithoutAttempt: true,
});
}
- if (admission.outcome === 'retired') {
+ if (admission.outcome === 'retired' || admission.outcome === 'order_unavailable') {
return { status: 'settled' };
}
+ const effectivePredecessorCreatedAt = admission.turn.admissionEffectivePredecessorCreatedAt;
+ const lineagePredecessorId = admission.turn.admissionLineagePredecessorId;
return {
status: 'ready',
@@ -626,6 +649,8 @@ function createAgentQueuedTurnResolver({
sourceId: claim.queuedTurnId,
claimId: claim.claimId,
claimBy: claim.claimBy,
+ ...(effectivePredecessorCreatedAt != null && { effectivePredecessorCreatedAt }),
+ ...(lineagePredecessorId != null && { lineagePredecessorId }),
},
releaseOnDefiniteFailure: async (error) => {
if (
@@ -659,6 +684,8 @@ function createAgentQueuedTurnResolver({
...(result.generationCreatedAt != null && {
generationCreatedAt: result.generationCreatedAt,
}),
+ ...(effectivePredecessorCreatedAt != null && { effectivePredecessorCreatedAt }),
+ ...(lineagePredecessorId != null && { lineagePredecessorId }),
settledAt: new Date(now()),
});
if (settled.outcome === 'conflict') {
@@ -794,7 +821,10 @@ function createAgentQueuedTurnScheduler({
}),
);
try {
- if (turn.status === 'claimed') {
+ const isIndeterminate =
+ turn.terminalReceipt?.outcome === 'dead' &&
+ turn.terminalReceipt.failure?.code === 'ADMISSION_INDETERMINATE';
+ if (turn.status === 'claimed' && !isIndeterminate) {
const result = await runAsSystem(() =>
methods.deadLetterAgentQueuedTurn({
user: turn.user,
diff --git a/packages/api/src/agents/triggers/host.ts b/packages/api/src/agents/triggers/host.ts
index d460f6ae7e..d62d0adf57 100644
--- a/packages/api/src/agents/triggers/host.ts
+++ b/packages/api/src/agents/triggers/host.ts
@@ -43,6 +43,8 @@ export interface AgentContinuationAdmissionSource {
sourceId: string;
claimId: string;
claimBy: string;
+ effectivePredecessorCreatedAt?: number;
+ lineagePredecessorId?: string;
}
export type AgentTriggerContinuePreparation =
diff --git a/packages/data-provider/src/types/queuedTurns.spec.ts b/packages/data-provider/src/types/queuedTurns.spec.ts
index 03c5a4fe69..b1ab7b4dc7 100644
--- a/packages/data-provider/src/types/queuedTurns.spec.ts
+++ b/packages/data-provider/src/types/queuedTurns.spec.ts
@@ -94,6 +94,34 @@ describe('agent queued turn schemas', () => {
expect(agentQueuedTurnReceiptSchema.parse(receipt)).toEqual(receipt);
});
+ it('parses the effective boundary consumed by an admitted turn', () => {
+ const receipt = {
+ ...request,
+ queuedTurnId: 'turn-1',
+ status: 'admitted',
+ effectivePredecessorCreatedAt: 84,
+ revision: 2,
+ createdAt: '2026-08-30T12:00:00.000Z',
+ updatedAt: '2026-08-30T12:01:00.000Z',
+ };
+
+ expect(agentQueuedTurnReceiptSchema.parse(receipt)).toEqual(receipt);
+ });
+
+ it('parses an explicit root admission without a synthetic timestamp', () => {
+ const receipt = {
+ ...request,
+ queuedTurnId: 'turn-root',
+ status: 'admitted',
+ rootPredecessor: true,
+ revision: 2,
+ createdAt: '2026-08-30T12:00:00.000Z',
+ updatedAt: '2026-08-30T12:01:00.000Z',
+ };
+
+ expect(agentQueuedTurnReceiptSchema.parse(receipt)).toEqual(receipt);
+ });
+
it('requires durability only when the capability is supported', () => {
expect(agentQueuedTurnCapabilitySchema.parse({ supported: false })).toEqual({
supported: false,
diff --git a/packages/data-provider/src/types/queuedTurns.ts b/packages/data-provider/src/types/queuedTurns.ts
index 5e7ef7daa0..19fe686dce 100644
--- a/packages/data-provider/src/types/queuedTurns.ts
+++ b/packages/data-provider/src/types/queuedTurns.ts
@@ -52,7 +52,13 @@ export type TCancelAgentQueuedTurnRequest = z.infer<typeof cancelAgentQueuedTurn
export const agentQueuedTurnReceiptSchema = enqueueAgentQueuedTurnSchema.extend({
queuedTurnId: z.string().trim().min(1),
status: z.enum(agentQueuedTurnStatuses),
+ /** Effective generation boundary consumed by an admitted turn. This can
+ * advance beyond the originally captured root as queued turns chain. */
+ effectivePredecessorCreatedAt: z.number().int().nonnegative().optional(),
+ /** Explicitly proves that this admission consumed no predecessor boundary. */
+ rootPredecessor: z.literal(true).optional(),
position: z.number().int().nonnegative().optional(),
+ /** Immutable queue sequence retained as `revision` for wire compatibility. */
revision: z.number().int().nonnegative(),
failure: z
.object({
diff --git a/packages/data-schemas/src/methods/queuedTurn.spec.ts b/packages/data-schemas/src/methods/queuedTurn.spec.ts
index a30965aada..b46b81f715 100644
--- a/packages/data-schemas/src/methods/queuedTurn.spec.ts
+++ b/packages/data-schemas/src/methods/queuedTurn.spec.ts
@@ -1,4 +1,5 @@
import mongoose from 'mongoose';
+import { createHash } from 'node:crypto';
import { MongoMemoryServer } from 'mongodb-memory-server';
import type { Model } from 'mongoose';
import type {
@@ -80,6 +81,10 @@ function claimInput(
};
}
+function rootLineageId(parentMessageId = 'parent-1') {
+ return `root:${createHash('sha256').update(parentMessageId).digest('base64url')}`;
+}
+
describe('agent queued turn methods', () => {
it('normalizes payloads, replays the same request, and conflicts on changed content', async () => {
const input = enqueueInput({
@@ -364,6 +369,280 @@ describe('agent queued turn methods', () => {
).resolves.toEqual({ outcome: 'blocked', claim: null });
});
+ it('serializes competing claims across independent method instances', async () => {
+ const normal = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'concurrent-claim-normal' }),
+ );
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'concurrent-claim-priority', priority: true }),
+ );
+ const otherReplica = createAgentQueuedTurnMethods(mongoose);
+
+ const [normalResult, priorityResult] = await Promise.all([
+ methods.claimNextAgentQueuedTurn(
+ claimInput(normal.turn.queuedTurnId, { claimId: 'normal-claim' }),
+ ),
+ otherReplica.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-claim',
+ claimBy: 'worker-2',
+ }),
+ ),
+ ]);
+
+ expect(normalResult).toEqual({ outcome: 'blocked', claim: null });
+ expect(priorityResult).toMatchObject({
+ outcome: 'acquired',
+ claim: { queuedTurnId: priority.turn.queuedTurnId },
+ });
+ await expect(Turn.countDocuments({ status: 'claimed' })).resolves.toBe(1);
+ });
+
+ it('keeps priority work behind an existing admission-order claim', async () => {
+ const normal = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'claimed-before-priority' }),
+ );
+ const normalClaim = claimInput(normal.turn.queuedTurnId, { claimId: 'normal-owner' });
+ await expect(methods.claimNextAgentQueuedTurn(normalClaim)).resolves.toMatchObject({
+ outcome: 'acquired',
+ });
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'priority-after-claim', priority: true }),
+ );
+ const otherReplica = createAgentQueuedTurnMethods(mongoose);
+
+ await expect(
+ otherReplica.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-owner',
+ claimBy: 'worker-2',
+ }),
+ ),
+ ).resolves.toEqual({ outcome: 'blocked', claim: null });
+
+ await methods.releaseAgentQueuedTurn({
+ ...normalClaim,
+ disposition: 'dead',
+ settledAt: START,
+ failure: { code: 'TEST_SETTLED', message: 'Release the lane reservation' },
+ });
+ await expect(
+ otherReplica.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-owner',
+ claimBy: 'worker-2',
+ }),
+ ),
+ ).resolves.toMatchObject({ outcome: 'acquired' });
+ });
+
+ it('keeps one durable admission owner after the lane-writer lease is stolen', async () => {
+ const normal = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'durable-owner-normal' }),
+ );
+ await expect(
+ methods.claimNextAgentQueuedTurn(
+ claimInput(normal.turn.queuedTurnId, {
+ claimId: 'durable-owner',
+ leaseUntil: new Date(START.getTime() + 1),
+ }),
+ ),
+ ).resolves.toMatchObject({ outcome: 'acquired' });
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'durable-owner-priority', priority: true }),
+ );
+
+ await Sequence.updateOne(
+ { user, tenantId: 'tenant-1', conversationId: 'conversation-1' },
+ {
+ $set: {
+ writerId: 'stolen-writer',
+ writerUntil: new Date(START.getTime() + 60_000),
+ },
+ },
+ );
+ const otherReplica = createAgentQueuedTurnMethods(mongoose);
+ await expect(
+ otherReplica.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-after-stolen-writer',
+ claimBy: 'worker-2',
+ now: new Date(START.getTime() + 2),
+ leaseUntil: LATER,
+ }),
+ ),
+ ).resolves.toEqual({ outcome: 'blocked', claim: null });
+ await expect(Turn.countDocuments({ admissionSlot: true })).resolves.toBe(1);
+ });
+
+ it('rejects a pre-slot replica claim through the legacy-written status shell', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'cross-version-first' }),
+ );
+ const second = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'cross-version-second', priority: true }),
+ );
+ await Turn.updateOne(
+ { _id: first.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ admissionSlot: true,
+ claimId: 'new-replica-claim',
+ claimBy: 'new-replica',
+ claimUntil: LATER,
+ },
+ },
+ );
+
+ await expect(
+ Turn.updateOne(
+ { _id: second.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ claimId: 'legacy-replica-claim',
+ claimBy: 'legacy-replica',
+ claimUntil: LATER,
+ },
+ },
+ ),
+ ).rejects.toMatchObject({ code: 11000 });
+ });
+
+ it('retires and fail-closes duplicate pre-fence admission lanes before index creation', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'duplicate-admission-first' }),
+ );
+ const second = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'duplicate-admission-second' }),
+ );
+ const third = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ conversationId: 'conversation-2',
+ clientRequestId: 'duplicate-claim-first',
+ }),
+ );
+ const fourth = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ conversationId: 'conversation-2',
+ clientRequestId: 'duplicate-claim-second',
+ }),
+ );
+ await Turn.collection.dropIndex('agent_queued_turn_claim_lane');
+ await Turn.collection.dropIndex('agent_queued_turn_admission_started_lane');
+ for (const [turn, deliveryKey, status] of [
+ [first, 'legacy-duplicate-first', 'dead'],
+ [second, 'legacy-duplicate-second', 'claimed'],
+ ] as const) {
+ await Turn.updateOne(
+ { _id: turn.turn.queuedTurnId },
+ {
+ $set: {
+ status,
+ deliveryKey,
+ deliveryState: 'published',
+ admissionId: deliveryKey,
+ admissionStartedAt: START,
+ ...(status === 'dead' && {
+ terminalReceipt: {
+ outcome: 'dead',
+ settledAt: START,
+ failure: {
+ code: 'ADMISSION_INDETERMINATE',
+ message: 'Legacy admission owner disappeared',
+ },
+ },
+ }),
+ },
+ },
+ );
+ }
+ for (const [turn, deliveryKey] of [
+ [third, 'legacy-claim-first'],
+ [fourth, 'legacy-claim-second'],
+ ] as const) {
+ await Turn.updateOne(
+ { _id: turn.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ deliveryKey,
+ deliveryState: 'published',
+ claimId: deliveryKey,
+ claimBy: 'legacy-worker',
+ claimUntil: LATER,
+ },
+ },
+ );
+ }
+
+ await expect(methods.ensureAgentQueuedTurnIndexes()).resolves.toBeUndefined();
+ await expect(
+ Sequence.findOne({ user, tenantId: 'tenant-1', conversationId: 'conversation-1' }).lean(),
+ ).resolves.toMatchObject({ retiredAt: expect.any(Date) });
+ await expect(
+ Sequence.findOne({ user, tenantId: 'tenant-1', conversationId: 'conversation-2' }).lean(),
+ ).resolves.toMatchObject({ retiredAt: expect.any(Date) });
+ await expect(
+ Turn.find({
+ _id: {
+ $in: [
+ first.turn.queuedTurnId,
+ second.turn.queuedTurnId,
+ third.turn.queuedTurnId,
+ fourth.turn.queuedTurnId,
+ ],
+ },
+ })
+ .sort({ sequence: 1 })
+ .lean(),
+ ).resolves.toMatchObject([
+ {
+ status: 'dead',
+ terminalReceipt: {
+ failure: { code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE' },
+ },
+ },
+ {
+ status: 'dead',
+ terminalReceipt: {
+ failure: { code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE' },
+ },
+ },
+ {
+ status: 'dead',
+ terminalReceipt: {
+ failure: { code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE' },
+ },
+ },
+ {
+ status: 'dead',
+ terminalReceipt: {
+ failure: { code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE' },
+ },
+ },
+ ]);
+ await expect(Turn.countDocuments({ admissionStartedAt: { $exists: true } })).resolves.toBe(0);
+ await expect(
+ methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'duplicate-admission-follow-up' }),
+ ),
+ ).rejects.toBeInstanceOf(AgentQueuedTurnLaneRetiredError);
+ await expect(
+ methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ conversationId: 'conversation-2',
+ clientRequestId: 'duplicate-claim-follow-up',
+ }),
+ ),
+ ).rejects.toBeInstanceOf(AgentQueuedTurnLaneRetiredError);
+ await expect(Turn.collection.indexExists('agent_queued_turn_claim_lane')).resolves.toBe(true);
+ await expect(
+ Turn.collection.indexExists('agent_queued_turn_admission_started_lane'),
+ ).resolves.toBe(true);
+ });
+
it('reclaims an expired exact lease but never replays a different claim identity', async () => {
const queued = await methods.enqueueAgentQueuedTurn(enqueueInput());
const first = await methods.claimNextAgentQueuedTurn(
@@ -373,6 +652,9 @@ describe('agent queued turn methods', () => {
}),
);
expect(first.outcome).toBe('acquired');
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'priority-after-expired-claim', priority: true }),
+ );
const reclaimed = await methods.claimNextAgentQueuedTurn(
claimInput(queued.turn.queuedTurnId, {
@@ -386,6 +668,16 @@ describe('agent queued turn methods', () => {
outcome: 'acquired',
claim: { attempts: 2 },
});
+ await expect(
+ methods.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-claim',
+ claimBy: 'worker-3',
+ now: new Date(START.getTime() + 2),
+ leaseUntil: LATER,
+ }),
+ ),
+ ).resolves.toEqual({ outcome: 'blocked', claim: null });
});
it('repairs delivery scheduling idempotently and leaves unscheduled replays discoverable', async () => {
@@ -600,6 +892,7 @@ describe('agent queued turn methods', () => {
clientRequestId: 'admitted',
text: 'admitted',
priority: true,
+ expectedPredecessorCreatedAt: 83,
}),
);
await methods.reserveAgentQueuedTurnDelivery({
@@ -624,6 +917,8 @@ describe('agent queued turn methods', () => {
admissionMode: 'ordinary' as const,
generationId: 'generation-1',
generationCreatedAt: 84,
+ effectivePredecessorCreatedAt: 83,
+ lineagePredecessorId: rootLineageId(),
settledAt: START,
};
await expect(
@@ -632,13 +927,19 @@ describe('agent queued turn methods', () => {
admissionId: 'admission-1',
startedAt: START,
}),
- ).resolves.toMatchObject({ outcome: 'started' });
+ ).resolves.toMatchObject({
+ outcome: 'started',
+ turn: { admissionEffectivePredecessorCreatedAt: 83 },
+ });
await expect(methods.markAgentQueuedTurnAdmitted(admissionInput)).resolves.toMatchObject({
outcome: 'admitted',
turn: {
status: 'admitted',
deliveryState: 'published',
- terminalReceipt: { admissionId: 'admission-1' },
+ terminalReceipt: {
+ admissionId: 'admission-1',
+ effectivePredecessorCreatedAt: 83,
+ },
},
});
await expect(
@@ -650,6 +951,7 @@ describe('agent queued turn methods', () => {
admissionId: 'admission-1',
generationId: 'generation-1',
generationCreatedAt: 84,
+ effectivePredecessorCreatedAt: 83,
}),
).resolves.toBe(true);
await expect(
@@ -661,6 +963,19 @@ describe('agent queued turn methods', () => {
admissionId: 'admission-1',
generationId: 'generation-1',
generationCreatedAt: 85,
+ effectivePredecessorCreatedAt: 83,
+ }),
+ ).resolves.toBe(false);
+ await expect(
+ methods.hasAgentQueuedTurnAdmissionReceipt({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: admitted.turn.queuedTurnId,
+ admissionId: 'admission-1',
+ generationId: 'generation-1',
+ generationCreatedAt: 84,
+ effectivePredecessorCreatedAt: 82,
}),
).resolves.toBe(false);
await expect(methods.markAgentQueuedTurnAdmitted(admissionInput)).resolves.toMatchObject({
@@ -737,7 +1052,7 @@ describe('agent queued turn methods', () => {
).resolves.toEqual([]);
});
- it('keeps an ambiguous admission fenced when its delivery exhausts', async () => {
+ it('keeps an ambiguous admission fenced when its predecessor boundary is unavailable', async () => {
const queued = await methods.enqueueAgentQueuedTurn(
enqueueInput({ clientRequestId: 'ambiguous-admission' }),
);
@@ -764,6 +1079,27 @@ describe('agent queued turn methods', () => {
admissionId: 'delivery-ambiguous-admission',
startedAt: START,
});
+ await Turn.updateOne(
+ { _id: queued.turn.queuedTurnId },
+ { $unset: { admissionLineagePredecessorId: 1 } },
+ );
+ const incompleteEvidence = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'ambiguous-admission-evidence' }),
+ );
+ await Turn.updateOne(
+ { _id: incompleteEvidence.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: START,
+ lineagePredecessorId: rootLineageId(),
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
await expect(
methods.claimNextAgentQueuedTurn(
@@ -800,16 +1136,60 @@ describe('agent queued turn methods', () => {
deliveryKey: 'delivery-ambiguous-admission',
settledAt: LATER,
failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'admission result unavailable' },
+ admissionEvidence: {
+ generationId: 'generation-without-predecessor-proof',
+ generationCreatedAt: 42,
+ },
}),
).resolves.toMatchObject({
outcome: 'admission_indeterminate',
turn: {
- status: 'dead',
+ status: 'claimed',
admissionId: 'delivery-ambiguous-admission',
deliveryState: 'published',
terminalReceipt: { failure: { code: 'ADMISSION_INDETERMINATE' } },
},
});
+ await expect(
+ methods.deadLetterAgentQueuedTurn({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: queued.turn.queuedTurnId,
+ deliveryKey: 'delivery-ambiguous-admission',
+ settledAt: new Date(LATER.getTime() + 1),
+ failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'duplicate terminal callback' },
+ }),
+ ).resolves.toMatchObject({ outcome: 'already_terminal' });
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'priority-behind-indeterminate', priority: true }),
+ );
+ await expect(
+ methods.claimNextAgentQueuedTurn(
+ claimInput(priority.turn.queuedTurnId, {
+ claimId: 'priority-behind-indeterminate',
+ claimBy: 'worker-2',
+ now: LATER,
+ leaseUntil: new Date(LATER.getTime() + 60_000),
+ }),
+ ),
+ ).resolves.toEqual({ outcome: 'blocked', claim: null });
+ await expect(Turn.findOne({ _id: queued.turn.queuedTurnId }).lean()).resolves.toMatchObject({
+ admissionSlot: true,
+ });
+ await expect(
+ Turn.updateOne(
+ { _id: priority.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ claimId: 'legacy-priority-claim',
+ claimBy: 'legacy-worker',
+ claimUntil: new Date(LATER.getTime() + 60_000),
+ },
+ },
+ ),
+ ).rejects.toMatchObject({ code: 11000 });
await expect(
methods.cancelAgentQueuedTurn({
user,
@@ -826,114 +1206,257 @@ describe('agent queued turn methods', () => {
leaseUntil: new Date(LATER.getTime() + 60_000),
}),
),
- ).resolves.toMatchObject({ outcome: 'missing' });
+ ).resolves.toMatchObject({ outcome: 'blocked' });
await expect(
methods.prepareAgentQueuedTurnConversationDeletion({
user,
targets: [{ conversationId: 'conversation-1', tenantId: 'tenant-1' }],
}),
).rejects.toThrow('admission must settle');
+
+ /** A legacy worker changes its owner shell to `dead` when quarantine
+ * begins. It may then claim a successor, but the common started-admission
+ * index must prevent that successor from crossing the provider boundary. */
+ await Turn.updateOne(
+ { _id: queued.turn.queuedTurnId },
+ { $set: { status: 'dead' }, $unset: { admissionSlot: 1 } },
+ );
+ await expect(
+ Turn.updateOne(
+ { _id: priority.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ claimId: 'legacy-priority-claim',
+ claimBy: 'legacy-worker',
+ claimUntil: new Date(LATER.getTime() + 60_000),
+ },
+ },
+ ),
+ ).resolves.toMatchObject({ modifiedCount: 1 });
+ await expect(
+ Turn.updateOne(
+ { _id: priority.turn.queuedTurnId },
+ {
+ $set: {
+ admissionId: 'legacy-priority-claim',
+ admissionStartedAt: new Date(LATER.getTime() + 1),
+ },
+ },
+ ),
+ ).rejects.toMatchObject({ code: 11000 });
});
- it('reconciles an ambiguous admission from durable generation evidence', async () => {
+ it('reconciles a root admission through explicit lineage without a timestamp boundary', async () => {
const queued = await methods.enqueueAgentQueuedTurn(
- enqueueInput({ clientRequestId: 'reconciled-admission' }),
+ enqueueInput({ clientRequestId: 'root-lineage-reconciliation' }),
);
+ const deliveryKey = 'root-lineage-delivery';
await methods.reserveAgentQueuedTurnDelivery({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
queuedTurnId: queued.turn.queuedTurnId,
- deliveryKey: 'delivery-reconciled-admission',
+ deliveryKey,
});
await methods.markQueuedTurnScheduled({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
queuedTurnId: queued.turn.queuedTurnId,
- deliveryKey: 'delivery-reconciled-admission',
+ deliveryKey,
scheduledAt: START,
});
- await methods.claimNextAgentQueuedTurn(
- claimInput(queued.turn.queuedTurnId, { claimId: 'delivery-reconciled-admission' }),
- );
- await methods.beginAgentQueuedTurnAdmission({
- ...claimInput(queued.turn.queuedTurnId, { claimId: 'delivery-reconciled-admission' }),
- admissionId: 'delivery-reconciled-admission',
- startedAt: START,
+ const queuedClaim = claimInput(queued.turn.queuedTurnId, { claimId: deliveryKey });
+ await methods.claimNextAgentQueuedTurn(queuedClaim);
+ await expect(
+ methods.beginAgentQueuedTurnAdmission({
+ ...queuedClaim,
+ admissionId: deliveryKey,
+ startedAt: START,
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'started',
+ turn: { admissionLineagePredecessorId: rootLineageId() },
});
-
await expect(
methods.deadLetterAgentQueuedTurn({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
queuedTurnId: queued.turn.queuedTurnId,
- deliveryKey: 'delivery-reconciled-admission',
+ deliveryKey,
settledAt: LATER,
failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'admission result unavailable' },
- admissionEvidence: {
- generationId: 'generation-reconciled',
- generationCreatedAt: 42,
- },
+ admissionEvidence: { generationId: 'root-generation', generationCreatedAt: 42 },
}),
).resolves.toMatchObject({
outcome: 'admission_reconciled',
turn: {
status: 'admitted',
terminalReceipt: {
- admissionId: 'delivery-reconciled-admission',
- generationId: 'generation-reconciled',
+ generationId: 'root-generation',
generationCreatedAt: 42,
+ lineagePredecessorId: rootLineageId(),
+ rootPredecessor: true,
},
},
});
});
- it('reconciles a quarantined admission after exact evidence arrives and unblocks its successor', async () => {
- const first = await methods.enqueueAgentQueuedTurn(
- enqueueInput({ clientRequestId: 'late-reconciled-admission' }),
+ it('reconstructs a pre-upgrade chained boundary during exact-evidence reconciliation', async () => {
+ const predecessor = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ clientRequestId: 'reconciled-predecessor',
+ expectedPredecessorCreatedAt: 40,
+ }),
);
- const successor = await methods.enqueueAgentQueuedTurn(
- enqueueInput({ clientRequestId: 'late-reconciled-successor' }),
+ const queued = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'reconciled-admission', expectedPredecessorCreatedAt: 40 }),
);
- const deliveryKey = 'delivery-late-reconciled-admission';
await methods.reserveAgentQueuedTurnDelivery({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
- queuedTurnId: first.turn.queuedTurnId,
- deliveryKey,
+ queuedTurnId: predecessor.turn.queuedTurnId,
+ deliveryKey: 'delivery-reconciled-predecessor',
});
await methods.markQueuedTurnScheduled({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
- queuedTurnId: first.turn.queuedTurnId,
- deliveryKey,
+ queuedTurnId: predecessor.turn.queuedTurnId,
+ deliveryKey: 'delivery-reconciled-predecessor',
scheduledAt: START,
});
- await methods.claimNextAgentQueuedTurn(
- claimInput(first.turn.queuedTurnId, { claimId: deliveryKey }),
- );
+ const predecessorClaim = claimInput(predecessor.turn.queuedTurnId, {
+ claimId: 'delivery-reconciled-predecessor',
+ });
+ await methods.claimNextAgentQueuedTurn(predecessorClaim);
await methods.beginAgentQueuedTurnAdmission({
- ...claimInput(first.turn.queuedTurnId, { claimId: deliveryKey }),
- admissionId: deliveryKey,
+ ...predecessorClaim,
+ admissionId: 'delivery-reconciled-predecessor',
startedAt: START,
- admissionProtocolVersion: 2,
});
- await methods.deadLetterAgentQueuedTurn({
+ await methods.markAgentQueuedTurnAdmitted({
+ ...predecessorClaim,
+ admissionId: 'delivery-reconciled-predecessor',
+ admissionMode: 'ordinary',
+ generationId: 'generation-predecessor',
+ generationCreatedAt: 41,
+ effectivePredecessorCreatedAt: 40,
+ lineagePredecessorId: rootLineageId(),
+ settledAt: START,
+ });
+ await methods.reserveAgentQueuedTurnDelivery({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
- queuedTurnId: first.turn.queuedTurnId,
- deliveryKey,
- settledAt: LATER,
- failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'admission result unavailable' },
+ queuedTurnId: queued.turn.queuedTurnId,
+ deliveryKey: 'delivery-reconciled-admission',
});
-
- await expect(
- methods.claimQueuedTurnsForAdmissionReconciliation({
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: queued.turn.queuedTurnId,
+ deliveryKey: 'delivery-reconciled-admission',
+ scheduledAt: START,
+ });
+ const admissionClaim = claimInput(queued.turn.queuedTurnId, {
+ claimId: 'delivery-reconciled-admission',
+ });
+ await expect(methods.claimNextAgentQueuedTurn(admissionClaim)).resolves.toMatchObject({
+ outcome: 'acquired',
+ });
+ await expect(
+ methods.beginAgentQueuedTurnAdmission({
+ ...admissionClaim,
+ admissionId: 'delivery-reconciled-admission',
+ startedAt: START,
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'started',
+ turn: { expectedPredecessorCreatedAt: 40 },
+ });
+
+ await expect(
+ methods.deadLetterAgentQueuedTurn({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: queued.turn.queuedTurnId,
+ deliveryKey: 'delivery-reconciled-admission',
+ settledAt: LATER,
+ failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'admission result unavailable' },
+ admissionEvidence: {
+ generationId: 'generation-reconciled',
+ generationCreatedAt: 42,
+ },
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'admission_reconciled',
+ turn: {
+ status: 'admitted',
+ terminalReceipt: {
+ admissionId: 'delivery-reconciled-admission',
+ generationId: 'generation-reconciled',
+ generationCreatedAt: 42,
+ effectivePredecessorCreatedAt: 41,
+ },
+ },
+ });
+ });
+
+ it('reconciles a quarantined admission after exact evidence arrives and unblocks its successor', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ clientRequestId: 'late-reconciled-admission',
+ expectedPredecessorCreatedAt: 41,
+ }),
+ );
+ const successor = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ clientRequestId: 'late-reconciled-successor',
+ expectedPredecessorCreatedAt: 41,
+ }),
+ );
+ const deliveryKey = 'delivery-late-reconciled-admission';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ scheduledAt: START,
+ });
+ await methods.claimNextAgentQueuedTurn(
+ claimInput(first.turn.queuedTurnId, { claimId: deliveryKey }),
+ );
+ await methods.beginAgentQueuedTurnAdmission({
+ ...claimInput(first.turn.queuedTurnId, { claimId: deliveryKey }),
+ admissionId: deliveryKey,
+ startedAt: START,
+ });
+ await methods.deadLetterAgentQueuedTurn({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ settledAt: LATER,
+ failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'admission result unavailable' },
+ });
+
+ await expect(
+ methods.claimQueuedTurnsForAdmissionReconciliation({
claimId: 'reconciliation-1',
claimBy: 'reconciler-1',
now: new Date(LATER.getTime() + 1),
@@ -975,6 +1498,95 @@ describe('agent queued turn methods', () => {
).resolves.toMatchObject({ outcome: 'acquired' });
});
+ it('keeps protocol-v2 job evidence indeterminate until the exact source receipt arrives', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'source-fenced-evidence' }),
+ );
+ const successor = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'source-fenced-successor' }),
+ );
+ const deliveryKey = 'source-fenced-delivery';
+ const claim = claimInput(first.turn.queuedTurnId, { claimId: deliveryKey });
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ scheduledAt: START,
+ });
+ await methods.claimNextAgentQueuedTurn(claim);
+ await methods.beginAgentQueuedTurnAdmission({
+ ...claim,
+ admissionId: deliveryKey,
+ startedAt: START,
+ admissionProtocolVersion: 2,
+ });
+
+ await expect(
+ methods.deadLetterAgentQueuedTurn({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey,
+ settledAt: LATER,
+ failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'source receipt unavailable' },
+ admissionEvidence: {
+ generationId: 'transient-job-generation',
+ generationCreatedAt: 43,
+ },
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'admission_indeterminate',
+ turn: {
+ status: 'claimed',
+ admissionProtocolVersion: 2,
+ admissionSlot: true,
+ terminalReceipt: { failure: { code: 'ADMISSION_INDETERMINATE' } },
+ },
+ });
+ await expect(
+ methods.claimNextAgentQueuedTurn(
+ claimInput(successor.turn.queuedTurnId, { claimId: 'source-fenced-successor' }),
+ ),
+ ).resolves.toMatchObject({ outcome: 'blocked' });
+
+ /** A protocol-v2 owner created before admission slots existed can already
+ * be in the legacy dead shell when its authoritative source receipt lands. */
+ await Turn.updateOne(
+ { _id: first.turn.queuedTurnId },
+ {
+ $set: { status: 'dead' },
+ $unset: { admissionSlot: 1, claimId: 1, claimBy: 1, claimUntil: 1 },
+ },
+ );
+
+ await expect(
+ methods.markAgentQueuedTurnAdmitted({
+ ...claim,
+ admissionId: deliveryKey,
+ admissionMode: 'ordinary',
+ generationId: 'exact-source-generation',
+ generationCreatedAt: 44,
+ lineagePredecessorId: rootLineageId(),
+ settledAt: new Date(LATER.getTime() + 1),
+ }),
+ ).resolves.toMatchObject({ outcome: 'admitted', turn: { status: 'admitted' } });
+ await expect(
+ methods.claimNextAgentQueuedTurn(
+ claimInput(successor.turn.queuedTurnId, { claimId: 'source-fenced-successor' }),
+ ),
+ ).resolves.toMatchObject({ outcome: 'acquired' });
+ });
+
it('leases reconciliation fairly while preserving a late exact receipt', async () => {
const quarantine = async (conversationId: string, clientRequestId: string) => {
const queued = await methods.enqueueAgentQueuedTurn(
@@ -1066,6 +1678,7 @@ describe('agent queued turn methods', () => {
admissionMode: 'ordinary',
generationId: 'generation-late-proof',
generationCreatedAt: 44,
+ lineagePredecessorId: rootLineageId(),
settledAt: new Date(claimNow.getTime() + 1),
}),
).resolves.toMatchObject({ outcome: 'admitted', turn: { status: 'admitted' } });
@@ -1079,7 +1692,11 @@ describe('agent queued turn methods', () => {
enqueueInput({ clientRequestId: 'root-a-2', expectedPredecessorCreatedAt: 10 }),
);
const laterRoot = await methods.enqueueAgentQueuedTurn(
- enqueueInput({ clientRequestId: 'root-b-1', expectedPredecessorCreatedAt: 20 }),
+ enqueueInput({
+ clientRequestId: 'root-b-1',
+ parentMessageId: 'parent-2',
+ expectedPredecessorCreatedAt: 10,
+ }),
);
await methods.reserveAgentQueuedTurnDelivery({
user,
@@ -1107,6 +1724,8 @@ describe('agent queued turn methods', () => {
admissionId: 'admission-root-a',
admissionMode: 'ordinary',
generationCreatedAt: 11,
+ effectivePredecessorCreatedAt: 10,
+ lineagePredecessorId: rootLineageId(),
settledAt: START,
});
@@ -1116,18 +1735,522 @@ describe('agent queued turn methods', () => {
tenantId: 'tenant-1',
conversationId: 'conversation-1',
sequence: sameRoot.turn.sequence,
+ rootParentMessageId: 'parent-1',
expectedPredecessorCreatedAt: 10,
}),
- ).resolves.toBe(11);
+ ).resolves.toEqual({
+ effectivePredecessorCreatedAt: 11,
+ lineagePredecessorId: first.turn.queuedTurnId,
+ });
await expect(
methods.getEffectiveAgentQueuedTurnPredecessor({
user,
tenantId: 'tenant-1',
conversationId: 'conversation-1',
sequence: laterRoot.turn.sequence,
- expectedPredecessorCreatedAt: 20,
+ rootParentMessageId: 'parent-2',
+ expectedPredecessorCreatedAt: 10,
+ }),
+ ).resolves.toEqual({
+ effectivePredecessorCreatedAt: 10,
+ lineagePredecessorId: rootLineageId('parent-2'),
+ });
+
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: sameRoot.turn.queuedTurnId,
+ deliveryKey: 'admission-root-a-2',
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: sameRoot.turn.queuedTurnId,
+ deliveryKey: 'admission-root-a-2',
+ scheduledAt: START,
+ });
+ const sameRootClaim = claimInput(sameRoot.turn.queuedTurnId, {
+ claimId: 'admission-root-a-2',
+ });
+ await methods.claimNextAgentQueuedTurn(sameRootClaim);
+ await methods.beginAgentQueuedTurnAdmission({
+ ...sameRootClaim,
+ admissionId: 'admission-root-a-2',
+ startedAt: START,
+ });
+ await methods.markAgentQueuedTurnAdmitted({
+ ...sameRootClaim,
+ admissionId: 'admission-root-a-2',
+ admissionMode: 'ordinary',
+ generationCreatedAt: 12,
+ effectivePredecessorCreatedAt: 11,
+ lineagePredecessorId: first.turn.queuedTurnId,
+ settledAt: START,
+ });
+ await expect(
+ methods.getAgentQueuedTurnByClientRequestId({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ clientRequestId: 'root-a-2',
+ }),
+ ).resolves.toMatchObject({
+ terminalReceipt: { effectivePredecessorCreatedAt: 11 },
+ });
+ });
+
+ it('uses queued-turn identity when consecutive generations share a timestamp', async () => {
+ const first = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'same-timestamp-first', expectedPredecessorCreatedAt: 10 }),
+ );
+ const second = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'same-timestamp-second', expectedPredecessorCreatedAt: 10 }),
+ );
+ const firstDelivery = 'same-timestamp-first-delivery';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey: firstDelivery,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: first.turn.queuedTurnId,
+ deliveryKey: firstDelivery,
+ scheduledAt: START,
+ });
+ const firstClaim = claimInput(first.turn.queuedTurnId, { claimId: firstDelivery });
+ await methods.claimNextAgentQueuedTurn(firstClaim);
+ await methods.beginAgentQueuedTurnAdmission({
+ ...firstClaim,
+ admissionId: firstDelivery,
+ startedAt: START,
+ });
+ await methods.markAgentQueuedTurnAdmitted({
+ ...firstClaim,
+ admissionId: firstDelivery,
+ admissionMode: 'ordinary',
+ generationCreatedAt: 10,
+ effectivePredecessorCreatedAt: 10,
+ lineagePredecessorId: rootLineageId(),
+ settledAt: START,
+ });
+
+ const secondDelivery = 'same-timestamp-second-delivery';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: second.turn.queuedTurnId,
+ deliveryKey: secondDelivery,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: second.turn.queuedTurnId,
+ deliveryKey: secondDelivery,
+ scheduledAt: LATER,
+ });
+ const secondClaim = claimInput(second.turn.queuedTurnId, {
+ claimId: secondDelivery,
+ now: LATER,
+ leaseUntil: new Date(LATER.getTime() + 60_000),
+ });
+ await methods.claimNextAgentQueuedTurn(secondClaim);
+ await expect(
+ methods.beginAgentQueuedTurnAdmission({
+ ...secondClaim,
+ admissionId: secondDelivery,
+ startedAt: LATER,
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'started',
+ turn: {
+ admissionEffectivePredecessorCreatedAt: 10,
+ admissionLineagePredecessorId: first.turn.queuedTurnId,
+ },
+ });
+ });
+
+ it('fails closed when an admitted predecessor lacks generation evidence', async () => {
+ const incomplete = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'incomplete-predecessor', expectedPredecessorCreatedAt: 10 }),
+ );
+ const target = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ clientRequestId: 'after-incomplete-predecessor',
+ expectedPredecessorCreatedAt: 10,
+ }),
+ );
+ await Turn.updateOne(
+ { _id: incomplete.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: START,
+ effectivePredecessorCreatedAt: 10,
+ lineagePredecessorId: rootLineageId(),
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+ const deliveryKey = 'after-incomplete-delivery';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: target.turn.queuedTurnId,
+ deliveryKey,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: target.turn.queuedTurnId,
+ deliveryKey,
+ scheduledAt: START,
+ });
+ const targetClaim = claimInput(target.turn.queuedTurnId, { claimId: deliveryKey });
+ await methods.claimNextAgentQueuedTurn(targetClaim);
+ await expect(
+ methods.beginAgentQueuedTurnAdmission({
+ ...targetClaim,
+ admissionId: deliveryKey,
+ startedAt: START,
+ }),
+ ).resolves.toMatchObject({
+ outcome: 'order_unavailable',
+ turn: {
+ status: 'dead',
+ terminalReceipt: {
+ failure: { code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE' },
+ },
+ },
+ });
+ });
+
+ it('derives the predecessor from admission order when priority overtakes sequence', async () => {
+ const normal = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'priority-normal', expectedPredecessorCreatedAt: 10 }),
+ );
+ const priority = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({
+ clientRequestId: 'priority-overtake',
+ expectedPredecessorCreatedAt: 10,
+ priority: true,
+ }),
+ );
+ const priorityDelivery = 'admission-priority-overtake';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: priority.turn.queuedTurnId,
+ deliveryKey: priorityDelivery,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: priority.turn.queuedTurnId,
+ deliveryKey: priorityDelivery,
+ scheduledAt: START,
+ });
+ const priorityClaim = claimInput(priority.turn.queuedTurnId, {
+ claimId: priorityDelivery,
+ });
+ await expect(methods.claimNextAgentQueuedTurn(priorityClaim)).resolves.toMatchObject({
+ outcome: 'acquired',
+ });
+ await methods.beginAgentQueuedTurnAdmission({
+ ...priorityClaim,
+ admissionId: priorityDelivery,
+ startedAt: START,
+ });
+ await methods.markAgentQueuedTurnAdmitted({
+ ...priorityClaim,
+ admissionId: priorityDelivery,
+ admissionMode: 'ordinary',
+ generationCreatedAt: 11,
+ effectivePredecessorCreatedAt: 10,
+ lineagePredecessorId: rootLineageId(),
+ settledAt: START,
+ });
+
+ await Turn.updateOne(
+ { _id: priority.turn.queuedTurnId },
+ {
+ $unset: {
+ 'terminalReceipt.effectivePredecessorCreatedAt': 1,
+ 'terminalReceipt.lineagePredecessorId': 1,
+ },
+ },
+ );
+ await expect(
+ methods.getEffectiveAgentQueuedTurnPredecessor({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ sequence: normal.turn.sequence,
+ rootParentMessageId: 'parent-1',
+ expectedPredecessorCreatedAt: 10,
+ allowLegacyPredecessorInference: true,
+ }),
+ ).resolves.toEqual({
+ effectivePredecessorCreatedAt: 11,
+ lineagePredecessorId: priority.turn.queuedTurnId,
+ });
+ await Turn.updateOne(
+ { _id: priority.turn.queuedTurnId },
+ {
+ $set: {
+ 'terminalReceipt.effectivePredecessorCreatedAt': 10,
+ 'terminalReceipt.lineagePredecessorId': rootLineageId(),
+ },
+ },
+ );
+
+ const normalDelivery = 'admission-priority-normal';
+ await methods.reserveAgentQueuedTurnDelivery({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: normal.turn.queuedTurnId,
+ deliveryKey: normalDelivery,
+ });
+ await methods.markQueuedTurnScheduled({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: normal.turn.queuedTurnId,
+ deliveryKey: normalDelivery,
+ scheduledAt: START,
+ });
+ const normalClaim = claimInput(normal.turn.queuedTurnId, { claimId: normalDelivery });
+ await expect(methods.claimNextAgentQueuedTurn(normalClaim)).resolves.toMatchObject({
+ outcome: 'acquired',
+ });
+ await methods.beginAgentQueuedTurnAdmission({
+ ...normalClaim,
+ admissionId: normalDelivery,
+ startedAt: LATER,
+ });
+ await methods.markAgentQueuedTurnAdmitted({
+ ...normalClaim,
+ admissionId: normalDelivery,
+ admissionMode: 'ordinary',
+ generationCreatedAt: 12,
+ effectivePredecessorCreatedAt: 11,
+ lineagePredecessorId: priority.turn.queuedTurnId,
+ settledAt: LATER,
+ });
+ await Turn.updateOne(
+ { _id: normal.turn.queuedTurnId },
+ { $unset: { 'terminalReceipt.effectivePredecessorCreatedAt': 1 } },
+ );
+
+ await expect(
+ methods.getAgentQueuedTurnByClientRequestId({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ clientRequestId: 'priority-normal',
+ }),
+ ).resolves.toMatchObject({
+ terminalReceipt: { effectivePredecessorCreatedAt: 11 },
+ });
+
+ await Turn.updateMany(
+ { _id: { $in: [priority.turn.queuedTurnId, normal.turn.queuedTurnId] } },
+ {
+ $unset: {
+ 'terminalReceipt.effectivePredecessorCreatedAt': 1,
+ 'terminalReceipt.lineagePredecessorId': 1,
+ },
+ },
+ );
+ const ambiguous = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'priority-ambiguous', expectedPredecessorCreatedAt: 10 }),
+ );
+ await expect(
+ methods.getEffectiveAgentQueuedTurnPredecessor({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ sequence: ambiguous.turn.sequence,
+ rootParentMessageId: 'parent-1',
+ expectedPredecessorCreatedAt: 10,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('rejects an inferred legacy edge that reconnects to an exact-edge tail', async () => {
+ const exact = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'cycle-exact', expectedPredecessorCreatedAt: 10 }),
+ );
+ const legacy = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'cycle-legacy', expectedPredecessorCreatedAt: 10 }),
+ );
+ const target = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'cycle-target', expectedPredecessorCreatedAt: 10 }),
+ );
+ await Turn.updateOne(
+ { _id: exact.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: START,
+ generationCreatedAt: 11,
+ effectivePredecessorCreatedAt: 10,
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+ await Turn.updateOne(
+ { _id: legacy.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: START,
+ generationCreatedAt: 11,
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+
+ await expect(
+ methods.getEffectiveAgentQueuedTurnPredecessor({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ sequence: target.turn.sequence,
+ rootParentMessageId: 'parent-1',
+ expectedPredecessorCreatedAt: 10,
+ allowLegacyPredecessorInference: true,
+ }),
+ ).resolves.toBeNull();
+ });
+
+ it('does not repair a legacy receipt through its successor edge', async () => {
+ const target = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'repair-target', expectedPredecessorCreatedAt: 10 }),
+ );
+ const successor = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'repair-successor', expectedPredecessorCreatedAt: 10 }),
+ );
+ await Turn.updateOne(
+ { _id: target.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: START,
+ generationCreatedAt: 10,
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+ await Turn.updateOne(
+ { _id: successor.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: LATER,
+ generationCreatedAt: 11,
+ effectivePredecessorCreatedAt: 10,
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+
+ const repaired = await methods.getAgentQueuedTurnByClientRequestId({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ clientRequestId: 'repair-target',
+ });
+ expect(repaired?.terminalReceipt?.effectivePredecessorCreatedAt).toBeUndefined();
+ });
+
+ it('keeps exact late evidence quarantined behind an ambiguous legacy successor', async () => {
+ const target = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'reconcile-target', expectedPredecessorCreatedAt: 10 }),
+ );
+ const successor = await methods.enqueueAgentQueuedTurn(
+ enqueueInput({ clientRequestId: 'reconcile-successor', expectedPredecessorCreatedAt: 10 }),
+ );
+ const deliveryKey = 'reconcile-target-delivery';
+ await Turn.updateOne(
+ { _id: target.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'claimed',
+ deliveryKey,
+ deliveryState: 'published',
+ claimId: deliveryKey,
+ claimBy: 'worker-1',
+ claimUntil: LATER,
+ admissionId: deliveryKey,
+ admissionStartedAt: START,
+ },
+ },
+ );
+ await Turn.updateOne(
+ { _id: successor.turn.queuedTurnId },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: LATER,
+ generationCreatedAt: 11,
+ effectivePredecessorCreatedAt: 10,
+ },
+ },
+ $unset: { activeSlot: 1 },
+ },
+ );
+
+ await expect(
+ methods.deadLetterAgentQueuedTurn({
+ user,
+ tenantId: 'tenant-1',
+ conversationId: 'conversation-1',
+ queuedTurnId: target.turn.queuedTurnId,
+ deliveryKey,
+ settledAt: LATER,
+ failure: { code: 'ATTEMPTS_EXHAUSTED', message: 'result unavailable' },
+ admissionEvidence: { generationCreatedAt: 10 },
}),
- ).resolves.toBeUndefined();
+ ).resolves.toMatchObject({
+ outcome: 'admission_indeterminate',
+ turn: {
+ status: 'claimed',
+ terminalReceipt: {
+ failure: { code: 'ADMISSION_INDETERMINATE' },
+ },
+ },
+ });
});
it('projects the newest failures when more than 100 dead receipts await dismissal', async () => {
@@ -1421,6 +2544,7 @@ describe('agent queued turn methods', () => {
admissionId: 'delivery-admission-during-delete',
admissionMode: 'ordinary',
generationCreatedAt: 42,
+ lineagePredecessorId: rootLineageId(),
settledAt: LATER,
});
await expect(
diff --git a/packages/data-schemas/src/methods/queuedTurn.ts b/packages/data-schemas/src/methods/queuedTurn.ts
index 83af0cfd38..35c4dae5b4 100644
--- a/packages/data-schemas/src/methods/queuedTurn.ts
+++ b/packages/data-schemas/src/methods/queuedTurn.ts
@@ -24,11 +24,25 @@ const LANE_WRITER_LEASE_MS = 30_000;
const LANE_WRITER_RETRY_MS = 5;
const LANE_WRITER_RETRIES = LANE_WRITER_LEASE_MS / LANE_WRITER_RETRY_MS;
const RETIRED_LANE_RETENTION_MS = 24 * 60 * 60_000;
+const ROOT_LINEAGE_PREDECESSOR_PREFIX = 'root:';
+const CLAIM_LANE_INDEX = 'agent_queued_turn_claim_lane';
+const ADMISSION_STARTED_LANE_INDEX = 'agent_queued_turn_admission_started_lane';
+const NAMESPACE_EXISTS = 48;
interface DuplicateKeyError {
code?: number;
}
+interface DuplicateAdmissionLane {
+ _id: {
+ tenantId?: string | null;
+ user: Types.ObjectId;
+ conversationId: string;
+ laneId: string;
+ };
+ count: number;
+}
+
export class AgentQueuedTurnConflictError extends Error {
constructor(clientRequestId: string) {
super(`Agent queued turn idempotency conflict: ${clientRequestId}`);
@@ -141,7 +155,10 @@ export type ClaimAgentQueuedTurnResult =
| { outcome: 'missing'; claim: null };
export type BeginAgentQueuedTurnAdmissionResult =
- | { outcome: 'started' | 'already_started' | 'retired'; turn: AgentQueuedTurnRecord }
+ | {
+ outcome: 'started' | 'already_started' | 'retired' | 'order_unavailable';
+ turn: AgentQueuedTurnRecord;
+ }
| { outcome: 'conflict'; turn: AgentQueuedTurnRecord | null };
export interface AgentQueuedTurnAdmissionEvidence {
@@ -149,6 +166,21 @@ export interface AgentQueuedTurnAdmissionEvidence {
generationCreatedAt: number;
}
+export interface AgentQueuedTurnPredecessor {
+ lineagePredecessorId: string;
+ effectivePredecessorCreatedAt?: number;
+}
+
+function getRootLineagePredecessorId(parentMessageId: string): string {
+ return `${ROOT_LINEAGE_PREDECESSOR_PREFIX}${createHash('sha256')
+ .update(requireBoundedString(parentMessageId, 256))
+ .digest('base64url')}`;
+}
+
+function isRootLineagePredecessorId(lineagePredecessorId: string): boolean {
+ return lineagePredecessorId.startsWith(ROOT_LINEAGE_PREDECESSOR_PREFIX);
+}
+
export interface ClaimAgentQueuedTurnReconciliationInput {
claimId: string;
claimBy: string;
@@ -237,6 +269,8 @@ export interface AgentQueuedTurnMethods {
admissionMode: 'warm' | 'ordinary';
generationId?: string;
generationCreatedAt?: number;
+ effectivePredecessorCreatedAt?: number;
+ lineagePredecessorId?: string;
settledAt: Date;
},
) => Promise<AdmitAgentQueuedTurnResult>;
@@ -246,6 +280,8 @@ export interface AgentQueuedTurnMethods {
admissionId: string;
generationId: string;
generationCreatedAt: number;
+ effectivePredecessorCreatedAt?: number;
+ lineagePredecessorId?: string;
},
) => Promise<boolean>;
deadLetterAgentQueuedTurn: (
@@ -262,9 +298,11 @@ export interface AgentQueuedTurnMethods {
getEffectiveAgentQueuedTurnPredecessor: (
input: AgentQueuedTurnConversationScope & {
sequence: number;
+ rootParentMessageId: string;
expectedPredecessorCreatedAt?: number;
+ allowLegacyPredecessorInference?: boolean;
},
- ) => Promise<number | undefined>;
+ ) => Promise<AgentQueuedTurnPredecessor | null>;
drainAgentQueuedTurns: (
input: AgentQueuedTurnOwnerScope & {
conversationId?: string;
@@ -536,6 +574,7 @@ function toRecord(turn: IAgentQueuedTurn): AgentQueuedTurnRecord {
...(turn.laneId != null && { laneId: turn.laneId }),
sequence: turn.sequence,
...(turn.activeSlot != null && { activeSlot: turn.activeSlot }),
+ ...(turn.admissionSlot === true && { admissionSlot: true }),
status: turn.status,
priority: turn.priority,
text: turn.text,
@@ -555,6 +594,12 @@ function toRecord(turn: IAgentQueuedTurn): AgentQueuedTurnRecord {
...(turn.claimUntil != null && { claimUntil: turn.claimUntil }),
...(turn.admissionId != null && { admissionId: turn.admissionId }),
...(turn.admissionStartedAt != null && { admissionStartedAt: turn.admissionStartedAt }),
+ ...(turn.admissionEffectivePredecessorCreatedAt != null && {
+ admissionEffectivePredecessorCreatedAt: turn.admissionEffectivePredecessorCreatedAt,
+ }),
+ ...(turn.admissionLineagePredecessorId != null && {
+ admissionLineagePredecessorId: turn.admissionLineagePredecessorId,
+ }),
...(turn.admissionProtocolVersion != null && {
admissionProtocolVersion: turn.admissionProtocolVersion,
}),
@@ -615,6 +660,7 @@ function requireClaim(turn: IAgentQueuedTurn | null): AgentQueuedTurnClaim | nul
if (
turn == null ||
turn.status !== 'claimed' ||
+ turn.admissionSlot !== true ||
turn.claimId == null ||
turn.claimBy == null ||
turn.claimUntil == null
@@ -655,8 +701,128 @@ export function createAgentQueuedTurnMethods(
}
}
+ /** Pre-fence replicas could start two admissions after the first owner moved
+ * to a legacy dead shell. Such a lane has no reconstructible total order.
+ * Retire it and terminalize every nonterminal row before installing the
+ * unique fence so upgrades remain available without guessing at ordering. */
+ async function quarantineDuplicateAdmissionLanes(): Promise<number> {
+ const lanes = await Turn().aggregate<DuplicateAdmissionLane>([
+ {
+ $match: {
+ $or: [{ status: 'claimed' }, { admissionStartedAt: { $exists: true } }],
+ },
+ },
+ {
+ $group: {
+ _id: {
+ tenantId: '$tenantId',
+ user: '$user',
+ conversationId: '$conversationId',
+ laneId: '$laneId',
+ },
+ count: { $sum: 1 },
+ },
+ },
+ { $match: { count: { $gt: 1 } } },
+ ]);
+ for (const lane of lanes) {
+ const scopeInput: AgentQueuedTurnConversationScope = {
+ user: lane._id.user,
+ ...(lane._id.tenantId != null && { tenantId: lane._id.tenantId }),
+ conversationId: requireBoundedString(lane._id.conversationId, 256),
+ };
+ const scope = conversationScope(scopeInput);
+ const laneId = requireBoundedString(lane._id.laneId, 128);
+ const settledAt = new Date();
+ await Sequence().updateOne(
+ { _id: laneKey(scopeInput), ...scope },
+ {
+ $set: { retiredAt: settledAt },
+ $setOnInsert: { ...conversationFields(scopeInput), laneId, value: 0 },
+ $unset: {
+ reservationId: 1,
+ writerId: 1,
+ writerUntil: 1,
+ expiresAt: 1,
+ },
+ },
+ { upsert: true },
+ );
+ await Turn().updateMany(
+ {
+ ...scope,
+ laneId,
+ $or: [
+ { status: { $in: ['reserving', 'queued', 'claimed'] } },
+ { admissionStartedAt: { $exists: true } },
+ ],
+ },
+ {
+ $set: {
+ status: 'dead',
+ terminalReceipt: {
+ outcome: 'dead',
+ settledAt,
+ failure: {
+ code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE',
+ message:
+ 'Multiple queued turns crossed the admission boundary without a durable order. Delete or replace this conversation before continuing.',
+ },
+ },
+ },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ admissionId: 1,
+ admissionStartedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
+ admissionProtocolVersion: 1,
+ reconciliationAvailableAt: 1,
+ reconciliationClaimId: 1,
+ reconciliationClaimBy: 1,
+ reconciliationClaimUntil: 1,
+ },
+ },
+ );
+ }
+ return lanes.length;
+ }
+
async function ensureAgentQueuedTurnIndexes(): Promise<void> {
- await Promise.all([createIndexesWithRetry(Turn()), createIndexesWithRetry(Sequence())]);
+ for (const model of [Turn(), Sequence()]) {
+ await model.createCollection().catch((error: unknown) => {
+ if ((error as DuplicateKeyError).code !== NAMESPACE_EXISTS) {
+ throw error;
+ }
+ });
+ }
+ const indexNames = new Set((await Turn().listIndexes()).map((index) => index.name));
+ if (!indexNames.has(CLAIM_LANE_INDEX) || !indexNames.has(ADMISSION_STARTED_LANE_INDEX)) {
+ for (let attempt = 0; ; attempt++) {
+ await quarantineDuplicateAdmissionLanes();
+ try {
+ await createIndexesWithRetry(Turn());
+ break;
+ } catch (error) {
+ const message = error instanceof Error ? error.message : String(error);
+ if (
+ attempt >= 4 ||
+ (error as DuplicateKeyError).code !== DUPLICATE_KEY ||
+ (!message.includes(CLAIM_LANE_INDEX) && !message.includes(ADMISSION_STARTED_LANE_INDEX))
+ ) {
+ throw error;
+ }
+ await new Promise((resolve) => setTimeout(resolve, 100 * 2 ** attempt));
+ }
+ }
+ } else {
+ await createIndexesWithRetry(Turn());
+ }
+ await createIndexesWithRetry(Sequence());
}
async function acquireLaneWriter(
@@ -996,6 +1162,78 @@ export function createAgentQueuedTurnMethods(
return turns.map(toActiveRecord);
}
+ async function ensureEffectiveAdmissionBoundary(
+ turn: IAgentQueuedTurn,
+ ): Promise<IAgentQueuedTurn> {
+ if (
+ turn._id == null ||
+ turn.sequence == null ||
+ turn.status !== 'admitted' ||
+ turn.terminalReceipt?.outcome !== 'admitted'
+ ) {
+ return turn;
+ }
+ const rootPredecessorCreatedAt = normalizePredecessor(turn.expectedPredecessorCreatedAt);
+ if (
+ turn.terminalReceipt.lineagePredecessorId != null &&
+ (turn.terminalReceipt.effectivePredecessorCreatedAt != null ||
+ (isRootLineagePredecessorId(turn.terminalReceipt.lineagePredecessorId) &&
+ turn.terminalReceipt.rootPredecessor === true))
+ ) {
+ return turn;
+ }
+ const resolvedPredecessor = await getEffectiveAgentQueuedTurnPredecessor({
+ user: turn.user,
+ ...(turn.tenantId != null && { tenantId: turn.tenantId }),
+ conversationId: turn.conversationId,
+ sequence: turn.sequence,
+ rootParentMessageId: turn.parentMessageId,
+ ...(rootPredecessorCreatedAt != null && {
+ expectedPredecessorCreatedAt: rootPredecessorCreatedAt,
+ }),
+ });
+ if (resolvedPredecessor === null) {
+ return turn;
+ }
+ const effectivePredecessorCreatedAt = resolvedPredecessor.effectivePredecessorCreatedAt;
+ await Turn().updateOne(
+ {
+ _id: turn._id,
+ status: 'admitted',
+ 'terminalReceipt.outcome': 'admitted',
+ $or: [
+ { 'terminalReceipt.effectivePredecessorCreatedAt': { $exists: false } },
+ { 'terminalReceipt.lineagePredecessorId': { $exists: false } },
+ { 'terminalReceipt.rootPredecessor': { $exists: false } },
+ ],
+ },
+ {
+ $set: {
+ ...(effectivePredecessorCreatedAt != null && {
+ 'terminalReceipt.effectivePredecessorCreatedAt': effectivePredecessorCreatedAt,
+ }),
+ 'terminalReceipt.lineagePredecessorId': resolvedPredecessor.lineagePredecessorId,
+ ...(effectivePredecessorCreatedAt == null &&
+ isRootLineagePredecessorId(resolvedPredecessor.lineagePredecessorId) && {
+ 'terminalReceipt.rootPredecessor': true,
+ }),
+ },
+ },
+ );
+ return {
+ ...turn,
+ terminalReceipt: {
+ ...turn.terminalReceipt,
+ ...(effectivePredecessorCreatedAt != null && { effectivePredecessorCreatedAt }),
+ lineagePredecessorId: resolvedPredecessor.lineagePredecessorId,
+ ...(effectivePredecessorCreatedAt == null &&
+ isRootLineagePredecessorId(resolvedPredecessor.lineagePredecessorId) && {
+ rootPredecessor: true as const,
+ }),
+ },
+ };
+ }
+
async function listAgentQueuedTurnReceipts(
input: AgentQueuedTurnConversationScope & { clientRequestIds?: readonly string[] },
): Promise<AgentQueuedTurnActiveRecord[]> {
@@ -1032,7 +1270,8 @@ export function createAgentQueuedTurnMethods(
turns.set(turn._id.toString(), turn);
}
}
- return [...turns.values()]
+ const repaired = await Promise.all([...turns.values()].map(ensureEffectiveAdmissionBoundary));
+ return repaired
.filter((turn) => turn.sequence != null)
.sort(
(left, right) =>
@@ -1051,7 +1290,7 @@ export function createAgentQueuedTurnMethods(
clientRequestId: requireBoundedString(input.clientRequestId, 128),
})
.lean<IAgentQueuedTurn>();
- return turn == null ? null : toRecord(turn);
+ return turn == null ? null : toRecord(await ensureEffectiveAdmissionBoundary(turn));
}
async function cancelAgentQueuedTurn(
@@ -1081,7 +1320,13 @@ export function createAgentQueuedTurnMethods(
status: 'cancelled',
terminalReceipt: { outcome: 'cancelled', settledAt },
},
- $unset: { activeSlot: 1, claimId: 1, claimBy: 1, claimUntil: 1 },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ },
},
{ new: true },
)
@@ -1219,6 +1464,11 @@ export function createAgentQueuedTurnMethods(
'terminalReceipt.outcome': 'dead',
'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
},
+ {
+ status: 'claimed',
+ 'terminalReceipt.outcome': 'dead',
+ 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
+ },
{ status: 'claimed', claimUntil: { $lte: input.now } },
],
},
@@ -1285,7 +1535,7 @@ export function createAgentQueuedTurnMethods(
...conversationScope(input),
_id: input.queuedTurnId,
deliveryKey: requireBoundedString(input.deliveryKey, 128),
- status: 'dead',
+ status: { $in: ['claimed', 'dead'] },
'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
reconciliationClaimId: requireBoundedString(input.claimId, 128),
reconciliationClaimBy: requireBoundedString(input.claimBy, 256),
@@ -1400,41 +1650,96 @@ export function createAgentQueuedTurnMethods(
throw new TypeError('Agent queued turn lease must end after claim time');
}
const scope = conversationScope(input);
- const replay = await Turn()
- .findOne({
- ...scope,
- _id: input.queuedTurnId,
- status: 'claimed',
- claimId,
- claimBy,
- claimUntil: { $gt: input.now },
- })
- .lean<IAgentQueuedTurn>();
- const replayedClaim = requireClaim(replay);
- if (replayedClaim != null) {
- return { outcome: 'replayed', claim: replayedClaim };
- }
-
const expected = await Turn()
.findOne({ ...scope, _id: input.queuedTurnId })
.lean<IAgentQueuedTurn>();
- if (expected == null || (expected.status !== 'queued' && expected.status !== 'claimed')) {
+ if (
+ expected == null ||
+ expected.laneId == null ||
+ (expected.status !== 'queued' && expected.status !== 'claimed')
+ ) {
return { outcome: 'missing', claim: null };
}
-
- const unfinishedReservation = await Turn().exists({ ...scope, status: 'reserving' });
- if (unfinishedReservation != null) {
+ const laneScope = { ...scope, laneId: expected.laneId };
+ const laneExists = await Sequence().exists({
+ _id: laneKey(input),
+ ...scope,
+ laneId: expected.laneId,
+ retiredAt: { $exists: false },
+ });
+ if (laneExists == null) {
+ return { outcome: 'missing', claim: null };
+ }
+ const competingReservation = await Turn().exists({
+ ...laneScope,
+ _id: { $ne: input.queuedTurnId },
+ $or: [
+ { admissionSlot: true },
+ // Pre-slot replicas and rows still expose their admission owner through
+ // the claimed status. Keep that legacy shell inside the same boundary.
+ { status: 'claimed' },
+ {
+ status: 'dead',
+ 'terminalReceipt.outcome': 'dead',
+ 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
+ },
+ ],
+ });
+ if (competingReservation != null) {
return { outcome: 'blocked', claim: null };
}
+ if (
+ expected.status === 'claimed' &&
+ expected.claimId === claimId &&
+ expected.claimBy === claimBy &&
+ expected.claimUntil != null &&
+ expected.claimUntil.getTime() > input.now.getTime()
+ ) {
+ if (expected.admissionSlot !== true) {
+ const migrated = await Turn()
+ .findOneAndUpdate(
+ {
+ ...laneScope,
+ _id: input.queuedTurnId,
+ status: 'claimed',
+ claimId,
+ claimBy,
+ admissionSlot: { $exists: false },
+ },
+ { $set: { admissionSlot: true } },
+ { new: true },
+ )
+ .lean<IAgentQueuedTurn>()
+ .catch((error: unknown) => {
+ if ((error as DuplicateKeyError).code === DUPLICATE_KEY) {
+ return null;
+ }
+ throw error;
+ });
+ const migratedClaim = requireClaim(migrated);
+ return migratedClaim == null
+ ? { outcome: 'blocked', claim: null }
+ : { outcome: 'replayed', claim: migratedClaim };
+ }
+ const replayedClaim = requireClaim(expected);
+ if (replayedClaim != null) {
+ return { outcome: 'replayed', claim: replayedClaim };
+ }
+ }
- const head = await Turn()
- .findOne({ ...scope, status: { $in: ['queued', 'claimed', 'dead'] } })
- .sort({ priority: -1, sequence: 1 })
- .lean<IAgentQueuedTurn>();
- if (head == null) {
+ const unfinishedReservation = await Turn().exists({ ...laneScope, status: 'reserving' });
+ if (unfinishedReservation != null) {
return { outcome: 'blocked', claim: null };
}
+ const head =
+ expected.status === 'claimed'
+ ? expected
+ : await Turn()
+ .findOne({ ...laneScope, status: { $in: ['queued', 'dead'] } })
+ .sort({ priority: -1, sequence: 1 })
+ .lean<IAgentQueuedTurn>();
if (
+ head == null ||
head._id?.toString() !== input.queuedTurnId ||
head.availableAt.getTime() > input.now.getTime() ||
(head.status === 'claimed' &&
@@ -1447,11 +1752,11 @@ export function createAgentQueuedTurnMethods(
const turn = await Turn()
.findOneAndUpdate(
{
- ...scope,
+ ...laneScope,
_id: input.queuedTurnId,
availableAt: { $lte: input.now },
$or: [
- { status: 'queued' },
+ { status: 'queued', admissionSlot: { $exists: false } },
{
status: 'claimed',
claimUntil: { $lte: input.now },
@@ -1462,6 +1767,7 @@ export function createAgentQueuedTurnMethods(
{
$set: {
status: 'claimed',
+ admissionSlot: true,
claimId,
claimBy,
claimUntil: input.leaseUntil,
@@ -1471,16 +1777,23 @@ export function createAgentQueuedTurnMethods(
},
{ new: true, sort: { priority: -1, sequence: 1 } },
)
- .lean<IAgentQueuedTurn>();
+ .lean<IAgentQueuedTurn>()
+ .catch((error: unknown) => {
+ if ((error as DuplicateKeyError).code === DUPLICATE_KEY) {
+ return null;
+ }
+ throw error;
+ });
const acquired = requireClaim(turn);
if (acquired != null) {
return { outcome: 'acquired', claim: acquired };
}
const racedReplay = await Turn()
.findOne({
- ...scope,
+ ...laneScope,
_id: input.queuedTurnId,
status: 'claimed',
+ admissionSlot: true,
claimId,
claimBy,
claimUntil: { $gt: input.now },
@@ -1518,8 +1831,12 @@ export function createAgentQueuedTurnMethods(
claimId: 1,
claimBy: 1,
claimUntil: 1,
+ admissionSlot: 1,
admissionId: 1,
admissionStartedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
+ admissionProtocolVersion: 1,
terminalReceipt: 1,
},
}
@@ -1537,11 +1854,15 @@ export function createAgentQueuedTurnMethods(
},
$unset: {
activeSlot: 1,
+ admissionSlot: 1,
claimId: 1,
claimBy: 1,
claimUntil: 1,
admissionId: 1,
admissionStartedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
+ admissionProtocolVersion: 1,
},
};
const turn = await Turn()
@@ -1596,7 +1917,13 @@ export function createAgentQueuedTurnMethods(
},
},
},
- $unset: { activeSlot: 1, claimId: 1, claimBy: 1, claimUntil: 1 },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ },
},
{ new: true },
)
@@ -1627,6 +1954,114 @@ export function createAgentQueuedTurnMethods(
throw error;
}
try {
+ const current = await Turn()
+ .findOne({ ...conversationScope(input), _id: input.queuedTurnId })
+ .lean<IAgentQueuedTurn>();
+ if (
+ current?.status === 'claimed' &&
+ current.claimId === input.claimId &&
+ current.claimBy === input.claimBy &&
+ current.admissionSlot === true &&
+ current.deliveryKey === admissionId &&
+ current.deliveryState === 'published' &&
+ current.laneId === writer.laneId &&
+ current.admissionId === admissionId &&
+ current.admissionStartedAt != null
+ ) {
+ return { outcome: 'already_started', turn: toRecord(current) };
+ }
+ if (
+ current?.status === 'claimed' &&
+ current.claimId === input.claimId &&
+ current.claimBy === input.claimBy &&
+ current.admissionStartedAt == null &&
+ current.laneId !== writer.laneId
+ ) {
+ const retired = await retireObsoleteClaim();
+ if (retired != null) {
+ return { outcome: 'retired', turn: retired };
+ }
+ }
+ if (
+ current?.status !== 'claimed' ||
+ current.claimId !== input.claimId ||
+ current.claimBy !== input.claimBy ||
+ current.admissionSlot !== true ||
+ current.deliveryKey !== admissionId ||
+ current.deliveryState !== 'published' ||
+ current.laneId !== writer.laneId ||
+ current.admissionId != null ||
+ current.sequence == null
+ ) {
+ return {
+ outcome: 'conflict',
+ turn: current == null ? null : toRecord(current),
+ };
+ }
+
+ const rootPredecessorCreatedAt = normalizePredecessor(current.expectedPredecessorCreatedAt);
+ const resolvedPredecessor = await getEffectiveAgentQueuedTurnPredecessor({
+ user: current.user,
+ ...(current.tenantId != null && { tenantId: current.tenantId }),
+ conversationId: current.conversationId,
+ sequence: current.sequence,
+ rootParentMessageId: current.parentMessageId,
+ ...(rootPredecessorCreatedAt != null && {
+ expectedPredecessorCreatedAt: rootPredecessorCreatedAt,
+ }),
+ allowLegacyPredecessorInference: true,
+ });
+ if (resolvedPredecessor === null) {
+ const dead = await Turn()
+ .findOneAndUpdate(
+ {
+ ...conversationScope(input),
+ _id: input.queuedTurnId,
+ status: 'claimed',
+ claimId: requireBoundedString(input.claimId, 128),
+ claimBy: requireBoundedString(input.claimBy, 256),
+ admissionSlot: true,
+ deliveryKey: admissionId,
+ deliveryState: 'published',
+ laneId: writer.laneId,
+ admissionId: { $exists: false },
+ },
+ {
+ $set: {
+ status: 'dead',
+ terminalReceipt: {
+ outcome: 'dead',
+ settledAt: input.startedAt,
+ failure: {
+ code: 'QUEUED_TURN_ADMISSION_ORDER_UNAVAILABLE',
+ message:
+ 'The queued turn admission order could not be reconstructed safely. Review this turn before sending it.',
+ },
+ },
+ },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ },
+ },
+ { new: true },
+ )
+ .lean<IAgentQueuedTurn>();
+ if (dead != null) {
+ return { outcome: 'order_unavailable', turn: toRecord(dead) };
+ }
+ const conflicted = await Turn()
+ .findOne({ ...conversationScope(input), _id: input.queuedTurnId })
+ .lean<IAgentQueuedTurn>();
+ return {
+ outcome: 'conflict',
+ turn: conflicted == null ? null : toRecord(conflicted),
+ };
+ }
+ const effectivePredecessorCreatedAt = resolvedPredecessor.effectivePredecessorCreatedAt;
const turn = await Turn()
.findOneAndUpdate(
{
@@ -1635,6 +2070,7 @@ export function createAgentQueuedTurnMethods(
status: 'claimed',
claimId: requireBoundedString(input.claimId, 128),
claimBy: requireBoundedString(input.claimBy, 256),
+ admissionSlot: true,
deliveryKey: admissionId,
deliveryState: 'published',
laneId: writer.laneId,
@@ -1644,6 +2080,10 @@ export function createAgentQueuedTurnMethods(
$set: {
admissionId,
admissionStartedAt: input.startedAt,
+ ...(effectivePredecessorCreatedAt != null && {
+ admissionEffectivePredecessorCreatedAt: effectivePredecessorCreatedAt,
+ }),
+ admissionLineagePredecessorId: resolvedPredecessor.lineagePredecessorId,
...(input.admissionProtocolVersion != null && {
admissionProtocolVersion: input.admissionProtocolVersion,
}),
@@ -1655,36 +2095,27 @@ export function createAgentQueuedTurnMethods(
if (turn != null) {
return { outcome: 'started', turn: toRecord(turn) };
}
- const current = await Turn()
+ const raced = await Turn()
.findOne({ ...conversationScope(input), _id: input.queuedTurnId })
.lean<IAgentQueuedTurn>();
if (
- current?.status === 'claimed' &&
- current.claimId === input.claimId &&
- current.claimBy === input.claimBy &&
- current.admissionStartedAt == null &&
- current.laneId !== writer.laneId
- ) {
- const retired = await retireObsoleteClaim();
- if (retired != null) {
- return { outcome: 'retired', turn: retired };
- }
- }
- if (
- current?.status === 'claimed' &&
- current.claimId === input.claimId &&
- current.claimBy === input.claimBy &&
- current.deliveryKey === admissionId &&
- current.deliveryState === 'published' &&
- current.laneId === writer.laneId &&
- current.admissionId === admissionId &&
- current.admissionStartedAt != null
+ raced?.status === 'claimed' &&
+ raced.claimId === input.claimId &&
+ raced.claimBy === input.claimBy &&
+ raced.admissionSlot === true &&
+ raced.deliveryKey === admissionId &&
+ raced.deliveryState === 'published' &&
+ raced.laneId === writer.laneId &&
+ raced.admissionId === admissionId &&
+ raced.admissionStartedAt != null &&
+ raced.admissionEffectivePredecessorCreatedAt === effectivePredecessorCreatedAt &&
+ raced.admissionLineagePredecessorId === resolvedPredecessor.lineagePredecessorId
) {
- return { outcome: 'already_started', turn: toRecord(current) };
+ return { outcome: 'already_started', turn: toRecord(raced) };
}
return {
outcome: 'conflict',
- turn: current == null ? null : toRecord(current),
+ turn: raced == null ? null : toRecord(raced),
};
} finally {
await releaseLaneWriter(input, writer.writerId);
@@ -1698,12 +2129,16 @@ export function createAgentQueuedTurnMethods(
admissionMode: 'warm' | 'ordinary';
generationId?: string;
generationCreatedAt?: number;
+ effectivePredecessorCreatedAt?: number;
+ lineagePredecessorId?: string;
settledAt: Date;
},
): Promise<AdmitAgentQueuedTurnResult> {
const admissionId = requireBoundedString(input.admissionId, 128);
const generationId = normalizeOptionalString(input.generationId, 256);
const generationCreatedAt = normalizePredecessor(input.generationCreatedAt);
+ const effectivePredecessorCreatedAt = normalizePredecessor(input.effectivePredecessorCreatedAt);
+ const lineagePredecessorId = normalizeOptionalString(input.lineagePredecessorId, 128);
const turn = await Turn()
.findOneAndUpdate(
{
@@ -1713,6 +2148,12 @@ export function createAgentQueuedTurnMethods(
deliveryState: 'published',
admissionId,
admissionStartedAt: { $exists: true },
+ ...(effectivePredecessorCreatedAt != null
+ ? { admissionEffectivePredecessorCreatedAt: effectivePredecessorCreatedAt }
+ : { admissionEffectivePredecessorCreatedAt: { $exists: false } }),
+ ...(lineagePredecessorId != null
+ ? { admissionLineagePredecessorId: lineagePredecessorId }
+ : { admissionLineagePredecessorId: { $exists: false } }),
$or: [
{
status: 'claimed',
@@ -1724,6 +2165,11 @@ export function createAgentQueuedTurnMethods(
'terminalReceipt.outcome': 'dead',
'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
},
+ {
+ status: 'claimed',
+ 'terminalReceipt.outcome': 'dead',
+ 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
+ },
],
},
{
@@ -1736,15 +2182,25 @@ export function createAgentQueuedTurnMethods(
admissionMode: input.admissionMode,
...(generationId != null && { generationId }),
...(generationCreatedAt != null && { generationCreatedAt }),
+ ...(effectivePredecessorCreatedAt != null && { effectivePredecessorCreatedAt }),
+ ...(lineagePredecessorId != null && { lineagePredecessorId }),
+ ...(effectivePredecessorCreatedAt == null &&
+ lineagePredecessorId != null &&
+ isRootLineagePredecessorId(lineagePredecessorId) && {
+ rootPredecessor: true,
+ }),
},
},
$unset: {
activeSlot: 1,
+ admissionSlot: 1,
claimId: 1,
claimBy: 1,
claimUntil: 1,
admissionId: 1,
admissionStartedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
admissionProtocolVersion: 1,
reconciliationAvailableAt: 1,
reconciliationClaimId: 1,
@@ -1780,9 +2236,13 @@ export function createAgentQueuedTurnMethods(
admissionId: string;
generationId: string;
generationCreatedAt: number;
+ effectivePredecessorCreatedAt?: number;
+ lineagePredecessorId?: string;
},
): Promise<boolean> {
const generationCreatedAt = normalizePredecessor(input.generationCreatedAt);
+ const effectivePredecessorCreatedAt = normalizePredecessor(input.effectivePredecessorCreatedAt);
+ const lineagePredecessorId = normalizeOptionalString(input.lineagePredecessorId, 128);
if (generationCreatedAt == null) {
throw new TypeError('Agent queued turn admission generation is invalid');
}
@@ -1795,6 +2255,12 @@ export function createAgentQueuedTurnMethods(
'terminalReceipt.admissionId': requireBoundedString(input.admissionId, 128),
'terminalReceipt.generationId': requireBoundedString(input.generationId, 256),
'terminalReceipt.generationCreatedAt': generationCreatedAt,
+ ...(effectivePredecessorCreatedAt != null && {
+ 'terminalReceipt.effectivePredecessorCreatedAt': effectivePredecessorCreatedAt,
+ }),
+ ...(lineagePredecessorId != null && {
+ 'terminalReceipt.lineagePredecessorId': lineagePredecessorId,
+ }),
})) != null
);
}
@@ -1818,58 +2284,126 @@ export function createAgentQueuedTurnMethods(
if (generationCreatedAt == null) {
throw new TypeError('Agent queued turn admission evidence is invalid');
}
- const reconciled = await Turn()
- .findOneAndUpdate(
- {
- ...scope,
- _id: input.queuedTurnId,
- deliveryKey,
- admissionId: deliveryKey,
- admissionStartedAt: { $exists: true },
- ...(input.reconciliationClaimId != null && {
- reconciliationClaimId: requireBoundedString(input.reconciliationClaimId, 128),
- }),
- ...(input.reconciliationClaimBy != null && {
- reconciliationClaimBy: requireBoundedString(input.reconciliationClaimBy, 256),
- }),
- $or: [
- { status: 'claimed' },
- {
- status: 'dead',
- 'terminalReceipt.outcome': 'dead',
- 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
- },
- ],
- },
- {
- $set: {
- status: 'admitted',
- terminalReceipt: {
- outcome: 'admitted',
- settledAt: input.settledAt,
- admissionId: deliveryKey,
- admissionMode: 'ordinary',
- ...(generationId != null && { generationId }),
- generationCreatedAt,
- },
- },
- $unset: {
- activeSlot: 1,
- claimId: 1,
- claimBy: 1,
- claimUntil: 1,
- admissionId: 1,
- admissionStartedAt: 1,
- admissionProtocolVersion: 1,
- reconciliationAvailableAt: 1,
- reconciliationClaimId: 1,
- reconciliationClaimBy: 1,
- reconciliationClaimUntil: 1,
- },
- },
- { new: true },
- )
+ const admission = await Turn()
+ .findOne({
+ ...scope,
+ _id: input.queuedTurnId,
+ deliveryKey,
+ admissionId: deliveryKey,
+ admissionStartedAt: { $exists: true },
+ })
+ .select({
+ sequence: 1,
+ parentMessageId: 1,
+ expectedPredecessorCreatedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
+ admissionProtocolVersion: 1,
+ })
.lean<IAgentQueuedTurn>();
+ let effectivePredecessorCreatedAt = normalizePredecessor(
+ admission?.admissionEffectivePredecessorCreatedAt,
+ );
+ let lineagePredecessorId = normalizeOptionalString(
+ admission?.admissionLineagePredecessorId,
+ 128,
+ );
+ if (admission?.sequence != null) {
+ const rootPredecessorCreatedAt = normalizePredecessor(
+ admission.expectedPredecessorCreatedAt,
+ );
+ const resolvedPredecessor = await getEffectiveAgentQueuedTurnPredecessor({
+ user: input.user,
+ ...(input.tenantId != null && { tenantId: input.tenantId }),
+ conversationId: input.conversationId,
+ sequence: admission.sequence,
+ rootParentMessageId: admission.parentMessageId,
+ ...(rootPredecessorCreatedAt != null && {
+ expectedPredecessorCreatedAt: rootPredecessorCreatedAt,
+ }),
+ });
+ const lineageConflict =
+ resolvedPredecessor === null ||
+ (lineagePredecessorId != null &&
+ lineagePredecessorId !== resolvedPredecessor.lineagePredecessorId) ||
+ (effectivePredecessorCreatedAt != null &&
+ effectivePredecessorCreatedAt !== resolvedPredecessor.effectivePredecessorCreatedAt);
+ if (lineageConflict) {
+ effectivePredecessorCreatedAt = undefined;
+ lineagePredecessorId = undefined;
+ } else {
+ effectivePredecessorCreatedAt = resolvedPredecessor.effectivePredecessorCreatedAt;
+ lineagePredecessorId = resolvedPredecessor.lineagePredecessorId;
+ }
+ }
+ /** GenerationJob evidence predates the source-receipt protocol. For v2,
+ * only the exact late source callback may prove provider admission. */
+ const reconciled =
+ admission?.admissionProtocolVersion === 2 || lineagePredecessorId == null
+ ? null
+ : await Turn()
+ .findOneAndUpdate(
+ {
+ ...scope,
+ _id: input.queuedTurnId,
+ deliveryKey,
+ admissionId: deliveryKey,
+ admissionStartedAt: { $exists: true },
+ ...(input.reconciliationClaimId != null && {
+ reconciliationClaimId: requireBoundedString(input.reconciliationClaimId, 128),
+ }),
+ ...(input.reconciliationClaimBy != null && {
+ reconciliationClaimBy: requireBoundedString(input.reconciliationClaimBy, 256),
+ }),
+ $or: [
+ { status: 'claimed' },
+ {
+ status: 'dead',
+ 'terminalReceipt.outcome': 'dead',
+ 'terminalReceipt.failure.code': 'ADMISSION_INDETERMINATE',
+ },
+ ],
+ },
+ {
+ $set: {
+ status: 'admitted',
+ terminalReceipt: {
+ outcome: 'admitted',
+ settledAt: input.settledAt,
+ admissionId: deliveryKey,
+ admissionMode: 'ordinary',
+ ...(generationId != null && { generationId }),
+ generationCreatedAt,
+ ...(effectivePredecessorCreatedAt != null && {
+ effectivePredecessorCreatedAt,
+ }),
+ lineagePredecessorId,
+ ...(effectivePredecessorCreatedAt == null &&
+ isRootLineagePredecessorId(lineagePredecessorId) && {
+ rootPredecessor: true,
+ }),
+ },
+ },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ admissionId: 1,
+ admissionStartedAt: 1,
+ admissionEffectivePredecessorCreatedAt: 1,
+ admissionLineagePredecessorId: 1,
+ admissionProtocolVersion: 1,
+ reconciliationAvailableAt: 1,
+ reconciliationClaimId: 1,
+ reconciliationClaimBy: 1,
+ reconciliationClaimUntil: 1,
+ },
+ },
+ { new: true },
+ )
+ .lean<IAgentQueuedTurn>();
if (reconciled != null) {
return { outcome: 'admission_reconciled', turn: toRecord(reconciled) };
}
@@ -1895,7 +2429,13 @@ export function createAgentQueuedTurnMethods(
},
},
},
- $unset: { activeSlot: 1, claimId: 1, claimBy: 1, claimUntil: 1 },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ },
},
{ new: true },
)
@@ -1912,7 +2452,8 @@ export function createAgentQueuedTurnMethods(
if (
current.status === 'claimed' &&
current.deliveryKey === deliveryKey &&
- current.admissionStartedAt != null
+ current.admissionStartedAt != null &&
+ current.terminalReceipt?.failure?.code !== 'ADMISSION_INDETERMINATE'
) {
const quarantined = await Turn()
.findOneAndUpdate(
@@ -1926,7 +2467,7 @@ export function createAgentQueuedTurnMethods(
},
{
$set: {
- status: 'dead',
+ status: 'claimed',
reconciliationAvailableAt: input.settledAt,
reconciliationAttempts: 0,
terminalReceipt: {
@@ -1954,7 +2495,12 @@ export function createAgentQueuedTurnMethods(
return { outcome: 'admission_indeterminate', turn: toRecord(quarantined) };
}
}
- if (['admitted', 'cancelled', 'dead'].includes(current.status)) {
+ if (
+ ['admitted', 'cancelled', 'dead'].includes(current.status) ||
+ (current.status === 'claimed' &&
+ current.terminalReceipt?.outcome === 'dead' &&
+ current.terminalReceipt.failure?.code === 'ADMISSION_INDETERMINATE')
+ ) {
return { outcome: 'already_terminal', turn: toRecord(current) };
}
return { outcome: 'conflict', turn: toRecord(current) };
@@ -1963,29 +2509,104 @@ export function createAgentQueuedTurnMethods(
async function getEffectiveAgentQueuedTurnPredecessor(
input: AgentQueuedTurnConversationScope & {
sequence: number;
+ rootParentMessageId: string;
expectedPredecessorCreatedAt?: number;
+ allowLegacyPredecessorInference?: boolean;
},
- ): Promise<number | undefined> {
+ ): Promise<AgentQueuedTurnPredecessor | null> {
if (!Number.isSafeInteger(input.sequence) || input.sequence <= 0) {
throw new TypeError('Agent queued turn sequence must be a positive integer');
}
const rootEpoch = normalizePredecessor(input.expectedPredecessorCreatedAt);
- if (rootEpoch == null) {
- return undefined;
- }
- const predecessor = await Turn()
- .findOne({
+ const rootParentMessageId = requireBoundedString(input.rootParentMessageId, 256);
+ const rootLineagePredecessorId = getRootLineagePredecessorId(rootParentMessageId);
+ const predecessors = await Turn()
+ .find({
...conversationScope(input),
- sequence: { $lt: input.sequence },
- expectedPredecessorCreatedAt: rootEpoch,
+ sequence: { $ne: input.sequence },
+ ...(rootEpoch == null
+ ? { expectedPredecessorCreatedAt: { $exists: false } }
+ : { expectedPredecessorCreatedAt: rootEpoch }),
+ parentMessageId: rootParentMessageId,
status: 'admitted',
'terminalReceipt.outcome': 'admitted',
- 'terminalReceipt.generationCreatedAt': { $exists: true },
})
- .sort({ sequence: -1 })
- .select('terminalReceipt.generationCreatedAt')
- .lean<IAgentQueuedTurn>();
- return predecessor?.terminalReceipt?.generationCreatedAt;
+ .select({ sequence: 1, terminalReceipt: 1 })
+ .lean<IAgentQueuedTurn[]>();
+ /** Queued-turn identities define lineage; timestamps are payload evidence,
+ * never graph nodes. One legacy row may be inserted at the proven tail
+ * while the caller owns admission. Missing output evidence, branches, and
+ * disconnected rows remain fail-closed. */
+ const edges = new Map<string, { queuedTurnId: string; generationCreatedAt: number }>();
+ const legacyRows: { queuedTurnId: string; generationCreatedAt: number }[] = [];
+ for (const turn of predecessors) {
+ const queuedTurnId = turn._id?.toString();
+ const generationCreatedAt = normalizePredecessor(turn.terminalReceipt?.generationCreatedAt);
+ if (queuedTurnId == null || generationCreatedAt == null) {
+ return null;
+ }
+ const lineagePredecessorId = normalizeOptionalString(
+ turn.terminalReceipt?.lineagePredecessorId,
+ 128,
+ );
+ if (lineagePredecessorId == null) {
+ legacyRows.push({ queuedTurnId, generationCreatedAt });
+ continue;
+ }
+ if (edges.has(lineagePredecessorId)) {
+ return null;
+ }
+ edges.set(lineagePredecessorId, { queuedTurnId, generationCreatedAt });
+ }
+ if (
+ legacyRows.length > 0 &&
+ (input.allowLegacyPredecessorInference !== true || legacyRows.length > 1)
+ ) {
+ return null;
+ }
+ let lineagePredecessorId = rootLineagePredecessorId;
+ let effectivePredecessorCreatedAt = rootEpoch;
+ let traversed = 0;
+ const visited = new Set<string>();
+ const advanceExactEdges = () => {
+ while (edges.has(lineagePredecessorId)) {
+ const edge = edges.get(lineagePredecessorId) as {
+ queuedTurnId: string;
+ generationCreatedAt: number;
+ };
+ if (visited.has(edge.queuedTurnId)) {
+ return false;
+ }
+ visited.add(edge.queuedTurnId);
+ lineagePredecessorId = edge.queuedTurnId;
+ effectivePredecessorCreatedAt = edge.generationCreatedAt;
+ traversed += 1;
+ }
+ return true;
+ };
+ if (!advanceExactEdges()) {
+ return null;
+ }
+ if (legacyRows.length === 1) {
+ const legacy = legacyRows[0];
+ if (visited.has(legacy.queuedTurnId)) {
+ return null;
+ }
+ visited.add(legacy.queuedTurnId);
+ lineagePredecessorId = legacy.queuedTurnId;
+ effectivePredecessorCreatedAt = legacy.generationCreatedAt;
+ traversed += 1;
+ if (!advanceExactEdges()) {
+ return null;
+ }
+ }
+ if (traversed !== predecessors.length) {
+ return null;
+ }
+ return {
+ lineagePredecessorId,
+ ...(effectivePredecessorCreatedAt != null && { effectivePredecessorCreatedAt }),
+ };
}
async function drainAgentQueuedTurns(
@@ -2016,7 +2637,13 @@ export function createAgentQueuedTurnMethods(
},
},
},
- $unset: { activeSlot: 1, claimId: 1, claimBy: 1, claimUntil: 1 },
+ $unset: {
+ activeSlot: 1,
+ admissionSlot: 1,
+ claimId: 1,
+ claimBy: 1,
+ claimUntil: 1,
+ },
},
);
await Turn().updateMany(
@@ -2144,6 +2771,7 @@ export function createAgentQueuedTurnMethods(
},
$unset: {
activeSlot: 1,
+ admissionSlot: 1,
reservationWriterId: 1,
claimId: 1,
claimBy: 1,
diff --git a/packages/data-schemas/src/schema/queuedTurn.ts b/packages/data-schemas/src/schema/queuedTurn.ts
index 22f9cc1a93..be0f2001ba 100644
--- a/packages/data-schemas/src/schema/queuedTurn.ts
+++ b/packages/data-schemas/src/schema/queuedTurn.ts
@@ -34,6 +34,9 @@ const terminalReceiptSchema = new Schema(
admissionMode: { type: String, enum: ['warm', 'ordinary'] },
generationId: { type: String, maxlength: 256 },
generationCreatedAt: { type: Number, min: 0 },
+ effectivePredecessorCreatedAt: { type: Number, min: 0 },
+ lineagePredecessorId: { type: String, maxlength: 128 },
+ rootPredecessor: { type: Boolean, enum: [true] },
failure: { type: failureSchema },
},
{ _id: false },
@@ -57,6 +60,7 @@ const queuedTurnSchema: Schema<IAgentQueuedTurnDocument> = new Schema(
sequence: { type: Number, min: 1 },
reservationWriterId: { type: String, maxlength: 128 },
activeSlot: { type: Number, min: 0, max: 99 },
+ admissionSlot: { type: Boolean },
status: {
type: String,
enum: ['reserving', 'queued', 'claimed', 'admitted', 'cancelled', 'dead'],
@@ -84,6 +88,8 @@ const queuedTurnSchema: Schema<IAgentQueuedTurnDocument> = new Schema(
claimUntil: { type: Date },
admissionStartedAt: { type: Date },
admissionId: { type: String, maxlength: 128 },
+ admissionEffectivePredecessorCreatedAt: { type: Number, min: 0 },
+ admissionLineagePredecessorId: { type: String, maxlength: 128 },
admissionProtocolVersion: { type: Number, enum: [2] },
reconciliationAvailableAt: { type: Date },
reconciliationClaimId: { type: String, maxlength: 128 },
@@ -115,6 +121,33 @@ queuedTurnSchema.index(
partialFilterExpression: { activeSlot: { $exists: true } },
},
);
+queuedTurnSchema.index(
+ { tenantId: 1, user: 1, conversationId: 1, laneId: 1, status: 1 },
+ {
+ name: 'agent_queued_turn_claim_lane',
+ unique: true,
+ partialFilterExpression: { status: 'claimed' },
+ },
+);
+queuedTurnSchema.index(
+ { tenantId: 1, user: 1, conversationId: 1, laneId: 1 },
+ {
+ name: 'agent_queued_turn_admission_started_lane',
+ unique: true,
+ /** `admissionStartedAt` is written by both legacy and current workers and
+ * survives legacy `dead/ADMISSION_INDETERMINATE` quarantine. This is the
+ * cross-version fence after a claim crosses the provider boundary. */
+ partialFilterExpression: { admissionStartedAt: { $exists: true } },
+ },
+);
+queuedTurnSchema.index(
+ { tenantId: 1, user: 1, conversationId: 1, laneId: 1, admissionSlot: 1 },
+ {
+ name: 'agent_queued_turn_admission_slot',
+ unique: true,
+ partialFilterExpression: { admissionSlot: true },
+ },
+);
queuedTurnSchema.index(
{
tenantId: 1,
diff --git a/packages/data-schemas/src/types/queuedTurn.ts b/packages/data-schemas/src/types/queuedTurn.ts
index ba9f47c75f..7d66a74d14 100644
--- a/packages/data-schemas/src/types/queuedTurn.ts
+++ b/packages/data-schemas/src/types/queuedTurn.ts
@@ -37,6 +37,12 @@ export interface AgentQueuedTurnTerminalReceipt {
admissionMode?: 'warm' | 'ordinary';
generationId?: string;
generationCreatedAt?: number;
+ /** Effective predecessor consumed by this admission after queued-turn chaining. */
+ effectivePredecessorCreatedAt?: number;
+ /** Durable lineage node: the root-message identity or predecessor queued-turn identity. */
+ lineagePredecessorId?: string;
+ /** This admission consumed no predecessor generation boundary. */
+ rootPredecessor?: true;
failure?: AgentQueuedTurnFailure;
}
@@ -57,6 +63,8 @@ export interface IAgentQueuedTurn {
reservationWriterId?: string;
/** Bounded active-lane capacity token. Present only while queued/claimed. */
activeSlot?: number;
+ /** Unique durable admission-order reservation for one conversation lane. */
+ admissionSlot?: boolean;
status: AgentQueuedTurnStatus;
priority: boolean;
text: string;
@@ -75,6 +83,10 @@ export interface IAgentQueuedTurn {
/** Durable proof that ordinary admission may have crossed the HTTP boundary. */
admissionStartedAt?: Date;
admissionId?: string;
+ /** Effective predecessor durably fenced before provider admission begins. */
+ admissionEffectivePredecessorCreatedAt?: number;
+ /** Durable lineage node fenced with the effective predecessor. */
+ admissionLineagePredecessorId?: string;
/** Version 2 requires accepted and deduplicated execution responses to prove
* the exact post-invocation source receipt. */
admissionProtocolVersion?: 2;
@@ -148,6 +160,7 @@ export type AgentQueuedTurnActiveRecord = Pick<
export interface AgentQueuedTurnClaim extends AgentQueuedTurnRecord {
status: 'claimed';
+ admissionSlot: true;
claimId: string;
claimBy: string;
claimUntil: Date;