Skip to content

Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active

🎛️ test: Guard Multi-Steer Injection Across Tool Boundaries (#14498)

Danny Avilaaa357a8e172b
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.js
index 32007dfd5c..b365937a7d 100644
--- a/e2e/setup/fake-model.js
+++ b/e2e/setup/fake-model.js
@@ -29,6 +29,8 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';
const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';
const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';
const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';
+const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';
+const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';
const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';
const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';
const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';
@@ -49,6 +51,8 @@ const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';
const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';
const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';
const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done';
+const STEER_SPLIT_FINAL_TEXT = 'E2E steer split reply done';
+const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';
const STEER_TOOL_NAME_PREFIX = 'remember_fact';
const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;
const SLOW_REPLY_CHUNKS = 160;
@@ -859,21 +863,139 @@ function steerToolReplyResponses(label, toolNames) {
],
};
}
- const chunks = Array.from(
+ let invocation = 0;
+ return {
+ responses: [''],
+ sleep: SLOW_CHUNK_DELAY_MS,
+ resolveInvocation: async (messages) => {
+ invocation += 1;
+ if (invocation === 1) {
+ return {
+ response: `E2E steer tool preamble ${label} ${slowChunkPayload()}`,
+ toolCalls: [
+ {
+ id: `call_e2e_steer_${label}`,
+ name: toolName,
+ args: { fact: `steer boundary ${label}` },
+ type: 'tool_call',
+ },
+ ],
+ };
+ }
+ return { response: `${STEER_TOOL_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };
+ },
+ };
+}
+
+/**
+ * Model-visible injection proof: echoes every steer-injected user message the
+ * model actually received (`additional_kwargs.source === 'steer'`, stamped by
+ * the SDK's `convertInjectedMessages`), so specs can assert the words reached
+ * the model rather than only that the UI rendered a part.
+ */
+function steerEchoSuffix(messages) {
+ const steerTexts = (messages ?? [])
+ .filter((message) => message?.additional_kwargs?.source === 'steer')
+ .map((message) => getContentText(message.content));
+ return `[steers-seen=${steerTexts.length}] ${steerTexts.join(' | ')}`.trim();
+}
+
+/** Slow word-chunk payload shared by the steer scenarios. */
+function slowChunkPayload() {
+ return Array.from(
{ length: SLOW_REPLY_CHUNKS },
(_, index) => `chunk-${String(index).padStart(3, '0')}`,
).join(' ');
+}
+
+/**
+ * Three-turn run with TWO tool boundaries for the split-steer e2e: turn 1
+ * streams a slow preamble then calls the MCP tool (boundary A), turn 2 streams
+ * a slow middle segment then calls it again (boundary B), turn 3 streams the
+ * final text. Lets a test land one steer before each boundary.
+ */
+function steerSplitReplyResponses(label, toolNames) {
+ const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));
+ if (!toolName) {
+ return {
+ responses: [
+ `E2E steer split reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,
+ ],
+ };
+ }
+ let invocation = 0;
return {
- responses: [`E2E steer tool preamble ${label} ${chunks}`, `${STEER_TOOL_FINAL_TEXT} ${label}`],
+ responses: [''],
sleep: SLOW_CHUNK_DELAY_MS,
- toolCalls: [
- {
- id: `call_e2e_steer_${label}`,
- name: toolName,
- args: { fact: `steer boundary ${label}` },
- type: 'tool_call',
- },
- ],
+ resolveInvocation: async (messages) => {
+ invocation += 1;
+ if (invocation === 1) {
+ return {
+ response: `E2E steer split preamble ${label} ${slowChunkPayload()}`,
+ toolCalls: [
+ {
+ id: `call_e2e_steer_split_a_${label}`,
+ name: toolName,
+ args: { fact: `steer split boundary A ${label}` },
+ type: 'tool_call',
+ },
+ ],
+ };
+ }
+ if (invocation === 2) {
+ return {
+ response: `E2E steer split middle ${label} ${slowChunkPayload()}`,
+ toolCalls: [
+ {
+ id: `call_e2e_steer_split_b_${label}`,
+ name: toolName,
+ args: { fact: `steer split boundary B ${label}` },
+ type: 'tool_call',
+ },
+ ],
+ };
+ }
+ return { response: `${STEER_SPLIT_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };
+ },
+ };
+}
+
+/**
+ * Two-turn run whose FINAL segment streams slowly: turn 1 streams a slow
+ * preamble then calls the MCP tool (the only boundary), turn 2 streams a slow
+ * final text. Lets a test submit a steer AFTER the last boundary — no drain
+ * point remains, so the terminal path must convert it to a queued follow-up.
+ */
+function steerLateReplyResponses(label, toolNames) {
+ const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));
+ if (!toolName) {
+ return {
+ responses: [
+ `E2E steer late reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,
+ ],
+ };
+ }
+ let invocation = 0;
+ return {
+ responses: [''],
+ sleep: SLOW_CHUNK_DELAY_MS,
+ resolveInvocation: async () => {
+ invocation += 1;
+ if (invocation === 1) {
+ return {
+ response: `E2E steer late preamble ${label} ${slowChunkPayload()}`,
+ toolCalls: [
+ {
+ id: `call_e2e_steer_late_${label}`,
+ name: toolName,
+ args: { fact: `steer late boundary ${label}` },
+ type: 'tool_call',
+ },
+ ],
+ };
+ }
+ return { response: `${STEER_LATE_FINAL_TEXT} ${label} ${slowChunkPayload()}` };
+ },
};
}
@@ -1436,6 +1558,16 @@ function resolveResponses({ graph, messages, text, toolNames }) {
return steerToolReplyResponses(steerToolLabel, toolNames);
}
+ const steerSplitLabel = getMarkerValue(text, STEER_SPLIT_REPLY_MARKER);
+ if (steerSplitLabel) {
+ return steerSplitReplyResponses(steerSplitLabel, toolNames);
+ }
+
+ const steerLateLabel = getMarkerValue(text, STEER_LATE_REPLY_MARKER);
+ if (steerLateLabel) {
+ return steerLateReplyResponses(steerLateLabel, toolNames);
+ }
+
if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {
return {
responses: [MOCK_REPLY],
diff --git a/e2e/specs/mock/steering.spec.ts b/e2e/specs/mock/steering.spec.ts
index fcb5c03734..e229a2ae6e 100644
--- a/e2e/specs/mock/steering.spec.ts
+++ b/e2e/specs/mock/steering.spec.ts
@@ -119,6 +119,11 @@ test.describe('mid-run steering and queuing', () => {
await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
timeout: 60000,
});
+ // Ordered content proof, not just a count: the echo carries the exact
+ // injected words in message order.
+ await expect(messagesView(page).getByText(`[steers-seen=1] ${steerText}`)).toBeVisible({
+ timeout: 30000,
+ });
// The steer stays INSIDE the response after run end — a user message at
// its injection point, not a queued follow-up turn (4 turns: the setup
@@ -129,6 +134,285 @@ test.describe('mid-run steering and queuing', () => {
await expect(queuedRows(page)).toHaveCount(0);
});
+ /**
+ * Two steers submitted in quick succession must BOTH inject at the next
+ * tool-batch boundary: the drain is an atomic take-all, the hook returns one
+ * injected message per item, and the host applies one content part per item.
+ * Regression: only one of two waiting steers went through.
+ */
+ test('steers twice in succession: both waiting bubbles inject at the same tool boundary', async ({
+ page,
+ }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('steer2');
+ const firstSteer = `First steer ${label}`;
+ const secondSteer = `Second steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, PROVIDER_C);
+ await selectEphemeralMCP(page);
+ await establishConversation(page, `steer2-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+
+ await typeDuringRun(page, firstSteer);
+ const [firstResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(firstResponse.status()).toBe(202);
+
+ await typeDuringRun(page, secondSteer);
+ const [secondResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(secondResponse.status()).toBe(202);
+
+ // Both steers wait as anchored bubbles — nothing injected yet.
+ await expect(inFlightSteers(page).filter({ hasText: firstSteer })).toHaveCount(1, {
+ timeout: 10000,
+ });
+ await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {
+ timeout: 10000,
+ });
+
+ // At the boundary, BOTH inject as in-thread parts, in submission order.
+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+ await expect(inFlightSteers(page)).toHaveCount(0);
+ await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
+ timeout: 60000,
+ });
+ // Model-visible proof: the fake model echoes the steer-injected user
+ // messages it actually received on the post-boundary turn — both unique
+ // texts, in submission order, so duplicated or swapped words fail here.
+ await expect(
+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
+ ).toBeVisible({ timeout: 30000 });
+
+ // Both survive run end inside the response — no queued follow-ups, no
+ // extra turns (setup pair + this pair).
+ await expect(messageTurns(page)).toHaveCount(4);
+ await expect(appliedSteerParts(page)).toHaveCount(2);
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
+ /**
+ * Human-cadence variant: the second steer is submitted while the FIRST
+ * steer's 202 is still pending, so the two POSTs (and their chip upserts)
+ * genuinely overlap in flight. The overlap is enforced, not raced: a route
+ * intercept forwards the first POST to the server (the enqueue happens in
+ * submission order) but holds its 202 from the client until the second POST
+ * has been observed. Both must still inject.
+ */
+ test('steers twice rapidly (second sent before the first ACK): both inject', async ({ page }) => {
+ test.setTimeout(150000);
+ const label = uniqueLabel('steerrapid');
+ const firstSteer = `Rapid first steer ${label}`;
+ const secondSteer = `Rapid second steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, PROVIDER_C);
+ await selectEphemeralMCP(page);
+ await establishConversation(page, `steerrapid-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+
+ let releaseFirstAck!: () => void;
+ const secondPosted = new Promise<void>((resolve) => (releaseFirstAck = resolve));
+ let steersSeen = 0;
+ let overlapProven = false;
+ await page.route('**/api/agents/chat/steer', async (route) => {
+ const ordinal = ++steersSeen;
+ if (ordinal === 2) {
+ releaseFirstAck();
+ }
+ // Forward to the real server first: the enqueue lands in submission
+ // order; only the CLIENT-side 202 delivery is held back.
+ const response = await route.fetch();
+ if (ordinal === 1) {
+ // Bounded so a broken second submit fails on assertions, not a hang.
+ await Promise.race([
+ secondPosted.then(() => {
+ overlapProven = true;
+ }),
+ new Promise<void>((resolve) => setTimeout(resolve, 10000)),
+ ]);
+ }
+ await route.fulfill({ response });
+ });
+
+ const steerResponseFor = (text: string) =>
+ page.waitForResponse(
+ (response) =>
+ isSteerRequest(response) && response.request().postData()?.includes(text) === true,
+ { timeout: 15000 },
+ );
+ const responses: Promise<Response>[] = [steerResponseFor(firstSteer)];
+ await typeDuringRun(page, firstSteer);
+ await messageInput(page).press('Enter');
+ responses.push(steerResponseFor(secondSteer));
+ await typeDuringRun(page, secondSteer);
+ await messageInput(page).press('Enter');
+
+ const [firstResponse, secondResponse] = await Promise.all(responses);
+ expect(firstResponse.status()).toBe(202);
+ expect(secondResponse.status()).toBe(202);
+ // The hold proves the overlap actually happened: the second POST was
+ // observed while the first 202 was still parked in the intercept.
+ expect(steersSeen).toBe(2);
+ expect(overlapProven).toBe(true);
+ await page.unroute('**/api/agents/chat/steer');
+
+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+ await expect(inFlightSteers(page)).toHaveCount(0);
+ await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({
+ timeout: 60000,
+ });
+ await expect(
+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
+ ).toBeVisible({ timeout: 30000 });
+ await expect(messageTurns(page)).toHaveCount(4);
+ await expect(appliedSteerParts(page)).toHaveCount(2);
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
+ /**
+ * Two steers split across DIFFERENT tool boundaries: the first drains at
+ * boundary A, the second is submitted while the next segment streams and
+ * must drain at boundary B. Regression guard for the succession case where
+ * a boundary falls between the two submissions.
+ */
+ test('steers split across two tool boundaries: each injects at its own boundary', async ({
+ page,
+ }) => {
+ test.setTimeout(180000);
+ const label = uniqueLabel('steersplit');
+ const firstSteer = `Boundary A steer ${label}`;
+ const secondSteer = `Boundary B steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, PROVIDER_C);
+ await selectEphemeralMCP(page);
+ await establishConversation(page, `steersplit-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_STEER_SPLIT_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+
+ // First steer lands during the turn-1 preamble.
+ await typeDuringRun(page, firstSteer);
+ const [firstResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(firstResponse.status()).toBe(202);
+
+ // Boundary A injects it while turn 2 is still ahead.
+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+
+ // Second steer lands during the turn-2 middle segment.
+ await typeDuringRun(page, secondSteer);
+ const [secondResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(secondResponse.status()).toBe(202);
+ await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {
+ timeout: 10000,
+ });
+
+ // Boundary B injects the second steer too.
+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+ await expect(inFlightSteers(page)).toHaveCount(0);
+ await expect(messagesView(page).getByText(`E2E steer split reply done ${label}`)).toBeVisible({
+ timeout: 60000,
+ });
+ // The post-boundary-B turn must have BOTH injected steers in its context,
+ // as the exact words in submission order.
+ await expect(
+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),
+ ).toBeVisible({ timeout: 30000 });
+
+ await expect(messageTurns(page)).toHaveCount(4);
+ await expect(appliedSteerParts(page)).toHaveCount(2);
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
+ /**
+ * A steer submitted AFTER the run's last tool boundary can never inject:
+ * the terminal drain reports it on the final event and the client must
+ * convert it to a queued follow-up and auto-send it as the next turn —
+ * the user's words go through either way, never silently dropped.
+ */
+ test('steer after the last tool boundary converts to a queued follow-up and auto-sends', async ({
+ page,
+ }) => {
+ test.setTimeout(180000);
+ const label = uniqueLabel('steerlate');
+ const firstSteer = `Injected steer ${label}`;
+ const lateSteer = `Late steer ${label}`;
+
+ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });
+ await selectMockEndpoint(page, PROVIDER_C);
+ await selectEphemeralMCP(page);
+ await establishConversation(page, `steerlate-setup-${label}`);
+
+ const run = await sendMessage(page, `E2E_STEER_LATE_REPLY:${label}`);
+ expect(run.ok()).toBeTruthy();
+
+ // First steer lands during the preamble and injects at the only boundary.
+ await typeDuringRun(page, firstSteer);
+ const [firstResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(firstResponse.status()).toBe(202);
+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {
+ timeout: 60000,
+ });
+
+ // The final segment is streaming now (its lead text is already visible) —
+ // this steer arrives after the last boundary.
+ await expect(messagesView(page).getByText(`E2E steer late reply done ${label}`)).toBeVisible({
+ timeout: 60000,
+ });
+ await typeDuringRun(page, lateSteer);
+ const [lateResponse] = await Promise.all([
+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),
+ messageInput(page).press('Enter'),
+ ]);
+ expect(lateResponse.status()).toBe(202);
+
+ // Never injected — converted to a queued follow-up at run end and
+ // auto-sent as the next user turn (6 turns: setup pair, this pair,
+ // auto-sent follow-up pair).
+ await expect(messageTurns(page)).toHaveCount(6, { timeout: 90000 });
+ const followupTurn = messageTurns(page).nth(4);
+ await expect(followupTurn).toContainText(lateSteer);
+ await expect(followupTurn.locator('.user-turn')).toBeVisible();
+ await expect(messageTurns(page).nth(5)).toContainText(MOCK_REPLY_TEXT, { timeout: 30000 });
+
+ await expect(appliedSteerParts(page)).toHaveCount(1);
+ await expect(inFlightSteers(page)).toHaveCount(0);
+ await expect(queuedRows(page)).toHaveCount(0);
+ });
+
test('queues with Cmd/Ctrl+Enter during a run and auto-sends after clean completion', async ({
page,
}) => {