Enhanced ChatGPT Clone: Features Agents, MCP, Skills, DeepSeek, Anthropic, AWS, OpenAI, Responses API, Azure, Groq, o1, GPT-5, Mistral, OpenRouter, Vertex AI, Gemini, Artifacts, AI model switching, message search, Code Interpreter, langchain, DALL-E-3, OpenAPI Actions, Functions, Secure Multi-User Auth, Presets, open-source for self-hosting. Active
TypeScript aianthropicartifactsawsazurechatgptchatgpt-cloneclaudeclonedeepseekgeminigooglegpt-5librechatmcpo1openairesponses-apivisionwebui
🎛️ test: Guard Multi-Steer Injection Across Tool Boundaries (#14498)
Danny Avilaaa357a8e172b
diff --git a/e2e/setup/fake-model.js b/e2e/setup/fake-model.jsindex 32007dfd5c..b365937a7d 100644--- a/e2e/setup/fake-model.js+++ b/e2e/setup/fake-model.js@@ -29,6 +29,8 @@ const COUNTED_REPLY_MARKER = 'E2E_COUNTED_REPLY:';const SLOW_REPLY_MARKER = 'E2E_SLOW_REPLY:';const SLOW_COUNTED_REPLY_MARKER = 'E2E_SLOW_COUNTED_REPLY:';const STEER_TOOL_REPLY_MARKER = 'E2E_STEER_TOOL_REPLY:';+const STEER_SPLIT_REPLY_MARKER = 'E2E_STEER_SPLIT_REPLY:';+const STEER_LATE_REPLY_MARKER = 'E2E_STEER_LATE_REPLY:';const RESUME_ICON_REPLY_MARKER = 'E2E_RESUME_ICON_REPLY:';const FORCED_ERROR_MARKER = 'E2E_FORCED_ERROR:';const MARKDOWN_REPLY_MARKER = 'E2E_MARKDOWN_REPLY';@@ -49,6 +51,8 @@ const PROVIDER_FILE_ASSERTION_FINAL_TEXT = 'E2E provider file assertion passed';const AGENT_CONTEXT_ASSERTION_FINAL_TEXT = 'E2E agent context assertion passed';const QUOTE_ASSERTION_FINAL_TEXT = 'E2E quote assertion passed';const STEER_TOOL_FINAL_TEXT = 'E2E steer tool reply done';+const STEER_SPLIT_FINAL_TEXT = 'E2E steer split reply done';+const STEER_LATE_FINAL_TEXT = 'E2E steer late reply done';const STEER_TOOL_NAME_PREFIX = 'remember_fact';const SLOW_CHUNK_DELAY_MS = Number(process.env.MOCK_LLM_SLOW_CHUNK_DELAY_MS) || 35;const SLOW_REPLY_CHUNKS = 160;@@ -859,21 +863,139 @@ function steerToolReplyResponses(label, toolNames) {],};}- const chunks = Array.from(+ let invocation = 0;+ return {+ responses: [''],+ sleep: SLOW_CHUNK_DELAY_MS,+ resolveInvocation: async (messages) => {+ invocation += 1;+ if (invocation === 1) {+ return {+ response: `E2E steer tool preamble ${label} ${slowChunkPayload()}`,+ toolCalls: [+ {+ id: `call_e2e_steer_${label}`,+ name: toolName,+ args: { fact: `steer boundary ${label}` },+ type: 'tool_call',+ },+ ],+ };+ }+ return { response: `${STEER_TOOL_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };+ },+ };+}++/**+ * Model-visible injection proof: echoes every steer-injected user message the+ * model actually received (`additional_kwargs.source === 'steer'`, stamped by+ * the SDK's `convertInjectedMessages`), so specs can assert the words reached+ * the model rather than only that the UI rendered a part.+ */+function steerEchoSuffix(messages) {+ const steerTexts = (messages ?? [])+ .filter((message) => message?.additional_kwargs?.source === 'steer')+ .map((message) => getContentText(message.content));+ return `[steers-seen=${steerTexts.length}] ${steerTexts.join(' | ')}`.trim();+}++/** Slow word-chunk payload shared by the steer scenarios. */+function slowChunkPayload() {+ return Array.from({ length: SLOW_REPLY_CHUNKS },(_, index) => `chunk-${String(index).padStart(3, '0')}`,).join(' ');+}++/**+ * Three-turn run with TWO tool boundaries for the split-steer e2e: turn 1+ * streams a slow preamble then calls the MCP tool (boundary A), turn 2 streams+ * a slow middle segment then calls it again (boundary B), turn 3 streams the+ * final text. Lets a test land one steer before each boundary.+ */+function steerSplitReplyResponses(label, toolNames) {+ const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));+ if (!toolName) {+ return {+ responses: [+ `E2E steer split reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,+ ],+ };+ }+ let invocation = 0;return {- responses: [`E2E steer tool preamble ${label} ${chunks}`, `${STEER_TOOL_FINAL_TEXT} ${label}`],+ responses: [''],sleep: SLOW_CHUNK_DELAY_MS,- toolCalls: [- {- id: `call_e2e_steer_${label}`,- name: toolName,- args: { fact: `steer boundary ${label}` },- type: 'tool_call',- },- ],+ resolveInvocation: async (messages) => {+ invocation += 1;+ if (invocation === 1) {+ return {+ response: `E2E steer split preamble ${label} ${slowChunkPayload()}`,+ toolCalls: [+ {+ id: `call_e2e_steer_split_a_${label}`,+ name: toolName,+ args: { fact: `steer split boundary A ${label}` },+ type: 'tool_call',+ },+ ],+ };+ }+ if (invocation === 2) {+ return {+ response: `E2E steer split middle ${label} ${slowChunkPayload()}`,+ toolCalls: [+ {+ id: `call_e2e_steer_split_b_${label}`,+ name: toolName,+ args: { fact: `steer split boundary B ${label}` },+ type: 'tool_call',+ },+ ],+ };+ }+ return { response: `${STEER_SPLIT_FINAL_TEXT} ${label} ${steerEchoSuffix(messages)}` };+ },+ };+}++/**+ * Two-turn run whose FINAL segment streams slowly: turn 1 streams a slow+ * preamble then calls the MCP tool (the only boundary), turn 2 streams a slow+ * final text. Lets a test submit a steer AFTER the last boundary — no drain+ * point remains, so the terminal path must convert it to a queued follow-up.+ */+function steerLateReplyResponses(label, toolNames) {+ const toolName = Array.from(toolNames).find((name) => name.startsWith(STEER_TOOL_NAME_PREFIX));+ if (!toolName) {+ return {+ responses: [+ `E2E steer late reply unavailable: no ${STEER_TOOL_NAME_PREFIX} tool advertised.`,+ ],+ };+ }+ let invocation = 0;+ return {+ responses: [''],+ sleep: SLOW_CHUNK_DELAY_MS,+ resolveInvocation: async () => {+ invocation += 1;+ if (invocation === 1) {+ return {+ response: `E2E steer late preamble ${label} ${slowChunkPayload()}`,+ toolCalls: [+ {+ id: `call_e2e_steer_late_${label}`,+ name: toolName,+ args: { fact: `steer late boundary ${label}` },+ type: 'tool_call',+ },+ ],+ };+ }+ return { response: `${STEER_LATE_FINAL_TEXT} ${label} ${slowChunkPayload()}` };+ },};}@@ -1436,6 +1558,16 @@ function resolveResponses({ graph, messages, text, toolNames }) {return steerToolReplyResponses(steerToolLabel, toolNames);}+ const steerSplitLabel = getMarkerValue(text, STEER_SPLIT_REPLY_MARKER);+ if (steerSplitLabel) {+ return steerSplitReplyResponses(steerSplitLabel, toolNames);+ }++ const steerLateLabel = getMarkerValue(text, STEER_LATE_REPLY_MARKER);+ if (steerLateLabel) {+ return steerLateReplyResponses(steerLateLabel, toolNames);+ }+if (text.includes(ASSERT_AGENT_CONTEXT_MARKER)) {return {responses: [MOCK_REPLY],diff --git a/e2e/specs/mock/steering.spec.ts b/e2e/specs/mock/steering.spec.tsindex fcb5c03734..e229a2ae6e 100644--- a/e2e/specs/mock/steering.spec.ts+++ b/e2e/specs/mock/steering.spec.ts@@ -119,6 +119,11 @@ test.describe('mid-run steering and queuing', () => {await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({timeout: 60000,});+ // Ordered content proof, not just a count: the echo carries the exact+ // injected words in message order.+ await expect(messagesView(page).getByText(`[steers-seen=1] ${steerText}`)).toBeVisible({+ timeout: 30000,+ });// The steer stays INSIDE the response after run end — a user message at// its injection point, not a queued follow-up turn (4 turns: the setup@@ -129,6 +134,285 @@ test.describe('mid-run steering and queuing', () => {await expect(queuedRows(page)).toHaveCount(0);});+ /**+ * Two steers submitted in quick succession must BOTH inject at the next+ * tool-batch boundary: the drain is an atomic take-all, the hook returns one+ * injected message per item, and the host applies one content part per item.+ * Regression: only one of two waiting steers went through.+ */+ test('steers twice in succession: both waiting bubbles inject at the same tool boundary', async ({+ page,+ }) => {+ test.setTimeout(150000);+ const label = uniqueLabel('steer2');+ const firstSteer = `First steer ${label}`;+ const secondSteer = `Second steer ${label}`;++ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await selectMockEndpoint(page, PROVIDER_C);+ await selectEphemeralMCP(page);+ await establishConversation(page, `steer2-setup-${label}`);++ const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);+ expect(run.ok()).toBeTruthy();++ await typeDuringRun(page, firstSteer);+ const [firstResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(firstResponse.status()).toBe(202);++ await typeDuringRun(page, secondSteer);+ const [secondResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(secondResponse.status()).toBe(202);++ // Both steers wait as anchored bubbles — nothing injected yet.+ await expect(inFlightSteers(page).filter({ hasText: firstSteer })).toHaveCount(1, {+ timeout: 10000,+ });+ await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {+ timeout: 10000,+ });++ // At the boundary, BOTH inject as in-thread parts, in submission order.+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {+ timeout: 60000,+ });+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {+ timeout: 60000,+ });+ await expect(inFlightSteers(page)).toHaveCount(0);+ await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({+ timeout: 60000,+ });+ // Model-visible proof: the fake model echoes the steer-injected user+ // messages it actually received on the post-boundary turn — both unique+ // texts, in submission order, so duplicated or swapped words fail here.+ await expect(+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),+ ).toBeVisible({ timeout: 30000 });++ // Both survive run end inside the response — no queued follow-ups, no+ // extra turns (setup pair + this pair).+ await expect(messageTurns(page)).toHaveCount(4);+ await expect(appliedSteerParts(page)).toHaveCount(2);+ await expect(queuedRows(page)).toHaveCount(0);+ });++ /**+ * Human-cadence variant: the second steer is submitted while the FIRST+ * steer's 202 is still pending, so the two POSTs (and their chip upserts)+ * genuinely overlap in flight. The overlap is enforced, not raced: a route+ * intercept forwards the first POST to the server (the enqueue happens in+ * submission order) but holds its 202 from the client until the second POST+ * has been observed. Both must still inject.+ */+ test('steers twice rapidly (second sent before the first ACK): both inject', async ({ page }) => {+ test.setTimeout(150000);+ const label = uniqueLabel('steerrapid');+ const firstSteer = `Rapid first steer ${label}`;+ const secondSteer = `Rapid second steer ${label}`;++ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await selectMockEndpoint(page, PROVIDER_C);+ await selectEphemeralMCP(page);+ await establishConversation(page, `steerrapid-setup-${label}`);++ const run = await sendMessage(page, `E2E_STEER_TOOL_REPLY:${label}`);+ expect(run.ok()).toBeTruthy();++ let releaseFirstAck!: () => void;+ const secondPosted = new Promise<void>((resolve) => (releaseFirstAck = resolve));+ let steersSeen = 0;+ let overlapProven = false;+ await page.route('**/api/agents/chat/steer', async (route) => {+ const ordinal = ++steersSeen;+ if (ordinal === 2) {+ releaseFirstAck();+ }+ // Forward to the real server first: the enqueue lands in submission+ // order; only the CLIENT-side 202 delivery is held back.+ const response = await route.fetch();+ if (ordinal === 1) {+ // Bounded so a broken second submit fails on assertions, not a hang.+ await Promise.race([+ secondPosted.then(() => {+ overlapProven = true;+ }),+ new Promise<void>((resolve) => setTimeout(resolve, 10000)),+ ]);+ }+ await route.fulfill({ response });+ });++ const steerResponseFor = (text: string) =>+ page.waitForResponse(+ (response) =>+ isSteerRequest(response) && response.request().postData()?.includes(text) === true,+ { timeout: 15000 },+ );+ const responses: Promise<Response>[] = [steerResponseFor(firstSteer)];+ await typeDuringRun(page, firstSteer);+ await messageInput(page).press('Enter');+ responses.push(steerResponseFor(secondSteer));+ await typeDuringRun(page, secondSteer);+ await messageInput(page).press('Enter');++ const [firstResponse, secondResponse] = await Promise.all(responses);+ expect(firstResponse.status()).toBe(202);+ expect(secondResponse.status()).toBe(202);+ // The hold proves the overlap actually happened: the second POST was+ // observed while the first 202 was still parked in the intercept.+ expect(steersSeen).toBe(2);+ expect(overlapProven).toBe(true);+ await page.unroute('**/api/agents/chat/steer');++ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {+ timeout: 60000,+ });+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {+ timeout: 60000,+ });+ await expect(inFlightSteers(page)).toHaveCount(0);+ await expect(messagesView(page).getByText(`E2E steer tool reply done ${label}`)).toBeVisible({+ timeout: 60000,+ });+ await expect(+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),+ ).toBeVisible({ timeout: 30000 });+ await expect(messageTurns(page)).toHaveCount(4);+ await expect(appliedSteerParts(page)).toHaveCount(2);+ await expect(queuedRows(page)).toHaveCount(0);+ });++ /**+ * Two steers split across DIFFERENT tool boundaries: the first drains at+ * boundary A, the second is submitted while the next segment streams and+ * must drain at boundary B. Regression guard for the succession case where+ * a boundary falls between the two submissions.+ */+ test('steers split across two tool boundaries: each injects at its own boundary', async ({+ page,+ }) => {+ test.setTimeout(180000);+ const label = uniqueLabel('steersplit');+ const firstSteer = `Boundary A steer ${label}`;+ const secondSteer = `Boundary B steer ${label}`;++ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await selectMockEndpoint(page, PROVIDER_C);+ await selectEphemeralMCP(page);+ await establishConversation(page, `steersplit-setup-${label}`);++ const run = await sendMessage(page, `E2E_STEER_SPLIT_REPLY:${label}`);+ expect(run.ok()).toBeTruthy();++ // First steer lands during the turn-1 preamble.+ await typeDuringRun(page, firstSteer);+ const [firstResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(firstResponse.status()).toBe(202);++ // Boundary A injects it while turn 2 is still ahead.+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {+ timeout: 60000,+ });++ // Second steer lands during the turn-2 middle segment.+ await typeDuringRun(page, secondSteer);+ const [secondResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(secondResponse.status()).toBe(202);+ await expect(inFlightSteers(page).filter({ hasText: secondSteer })).toHaveCount(1, {+ timeout: 10000,+ });++ // Boundary B injects the second steer too.+ await expect(appliedSteerParts(page).filter({ hasText: secondSteer })).toHaveCount(1, {+ timeout: 60000,+ });+ await expect(inFlightSteers(page)).toHaveCount(0);+ await expect(messagesView(page).getByText(`E2E steer split reply done ${label}`)).toBeVisible({+ timeout: 60000,+ });+ // The post-boundary-B turn must have BOTH injected steers in its context,+ // as the exact words in submission order.+ await expect(+ messagesView(page).getByText(`[steers-seen=2] ${firstSteer} | ${secondSteer}`),+ ).toBeVisible({ timeout: 30000 });++ await expect(messageTurns(page)).toHaveCount(4);+ await expect(appliedSteerParts(page)).toHaveCount(2);+ await expect(queuedRows(page)).toHaveCount(0);+ });++ /**+ * A steer submitted AFTER the run's last tool boundary can never inject:+ * the terminal drain reports it on the final event and the client must+ * convert it to a queued follow-up and auto-send it as the next turn —+ * the user's words go through either way, never silently dropped.+ */+ test('steer after the last tool boundary converts to a queued follow-up and auto-sends', async ({+ page,+ }) => {+ test.setTimeout(180000);+ const label = uniqueLabel('steerlate');+ const firstSteer = `Injected steer ${label}`;+ const lateSteer = `Late steer ${label}`;++ await page.goto(NEW_CHAT_PATH, { timeout: 10000 });+ await selectMockEndpoint(page, PROVIDER_C);+ await selectEphemeralMCP(page);+ await establishConversation(page, `steerlate-setup-${label}`);++ const run = await sendMessage(page, `E2E_STEER_LATE_REPLY:${label}`);+ expect(run.ok()).toBeTruthy();++ // First steer lands during the preamble and injects at the only boundary.+ await typeDuringRun(page, firstSteer);+ const [firstResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(firstResponse.status()).toBe(202);+ await expect(appliedSteerParts(page).filter({ hasText: firstSteer })).toHaveCount(1, {+ timeout: 60000,+ });++ // The final segment is streaming now (its lead text is already visible) —+ // this steer arrives after the last boundary.+ await expect(messagesView(page).getByText(`E2E steer late reply done ${label}`)).toBeVisible({+ timeout: 60000,+ });+ await typeDuringRun(page, lateSteer);+ const [lateResponse] = await Promise.all([+ page.waitForResponse(isSteerRequest, { timeout: 15000 }),+ messageInput(page).press('Enter'),+ ]);+ expect(lateResponse.status()).toBe(202);++ // Never injected — converted to a queued follow-up at run end and+ // auto-sent as the next user turn (6 turns: setup pair, this pair,+ // auto-sent follow-up pair).+ await expect(messageTurns(page)).toHaveCount(6, { timeout: 90000 });+ const followupTurn = messageTurns(page).nth(4);+ await expect(followupTurn).toContainText(lateSteer);+ await expect(followupTurn.locator('.user-turn')).toBeVisible();+ await expect(messageTurns(page).nth(5)).toContainText(MOCK_REPLY_TEXT, { timeout: 30000 });++ await expect(appliedSteerParts(page)).toHaveCount(1);+ await expect(inFlightSteers(page)).toHaveCount(0);+ await expect(queuedRows(page)).toHaveCount(0);+ });+test('queues with Cmd/Ctrl+Enter during a run and auto-sends after clean completion', async ({page,}) => {