An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(prompt): stop mutating the message array for related questions (#971)
Yoshiki Miura989a05697d97
diff --git a/lib/agents/__tests__/researcher.test.ts b/lib/agents/__tests__/researcher.test.tsindex 49ebe6d9..5e2c5981 100644--- a/lib/agents/__tests__/researcher.test.ts+++ b/lib/agents/__tests__/researcher.test.ts@@ -2,8 +2,6 @@ import { beforeEach, describe, expect, it, vi } from 'vitest'import type { Model } from '@/lib/types/models'-import { RELATED_QUESTIONS_REMINDER } from '../prompts/related-questions-reminder'-const mocks = vi.hoisted(() => ({agentOptions: undefined as Record<string, any> | undefined}))@@ -116,29 +114,4 @@ describe('createResearcher', () => {openai: { reasoningEffort: 'low' }})})-- it('places the related-questions reminder after conversation history', () => {- createResearcher({- model: 'openai:model-id',- modelConfig: createModel('openai')- })-- const prepareCall = mocks.agentOptions?.prepareCall- const result = prepareCall({- messages: [- { role: 'user', content: 'Initial question' },- { role: 'assistant', content: 'Initial answer' },- { role: 'user', content: 'Deep follow-up' }- ]- })-- expect(result.messages.at(-1)).toEqual({- role: 'user',- content: [{ type: 'text', text: RELATED_QUESTIONS_REMINDER }]- })- expect(result.messages.at(-2)).toEqual({- role: 'user',- content: 'Deep follow-up'- })- })})diff --git a/lib/agents/prompts/__tests__/related-questions-reminder.test.ts b/lib/agents/prompts/__tests__/related-questions-reminder.test.tsdeleted file mode 100644index 36d731f3..00000000--- a/lib/agents/prompts/__tests__/related-questions-reminder.test.ts+++ /dev/null@@ -1,142 +0,0 @@-import type { ModelMessage } from 'ai'-import { describe, expect, it } from 'vitest'--import {- appendRelatedQuestionsReminder,- RELATED_QUESTIONS_REMINDER-} from '../related-questions-reminder'--describe('appendRelatedQuestionsReminder', () => {- it('places the reminder in its own message after a deep conversation', () => {- const messages: ModelMessage[] = [- { role: 'user', content: 'First question' },- { role: 'assistant', content: 'First answer' },- { role: 'user', content: 'Follow-up question' }- ]- const originalMessages = structuredClone(messages)-- const result = appendRelatedQuestionsReminder(messages)-- expect(result).not.toBe(messages)- expect(result.at(-1)).toEqual({- role: 'user',- content: [{ type: 'text', text: RELATED_QUESTIONS_REMINDER }]- })- expect(result.at(-2)).toEqual(messages.at(-1))- expect(result.at(-2)).toBe(messages.at(-1))- expect(messages).toEqual(originalMessages)- })-- it('leaves a user message carrying non-text parts untouched', () => {- const messages: ModelMessage[] = [- {- role: 'user',- content: [- { type: 'text', text: 'Describe this file' },- {- type: 'file',- data: new Uint8Array([1, 2, 3]),- mediaType: 'application/pdf'- }- ]- }- ]- const originalMessage = messages[0]- const originalContent = originalMessage.content-- const result = appendRelatedQuestionsReminder(messages)-- expect(result[0]).toEqual(originalMessage)- expect(result[0]).toBe(originalMessage)- expect(result[1]).toEqual({- role: 'user',- content: [{ type: 'text', text: RELATED_QUESTIONS_REMINDER }]- })- expect(messages).toHaveLength(1)- expect(messages[0]).toBe(originalMessage)- expect(messages[0].content).toBe(originalContent)- expect(messages[0].content).toEqual([- { type: 'text', text: 'Describe this file' },- {- type: 'file',- data: new Uint8Array([1, 2, 3]),- mediaType: 'application/pdf'- }- ])- })-- it('leaves messages unchanged when there is no user message', () => {- const messages: ModelMessage[] = [- { role: 'assistant', content: 'Existing answer' }- ]-- expect(appendRelatedQuestionsReminder(messages)).toBe(messages)- })-- it('keeps replayed conversation history stable across turns', () => {- const turnNMessages: ModelMessage[] = [- { role: 'user', content: 'First question' },- { role: 'assistant', content: 'First answer' },- { role: 'user', content: 'Second question' }- ]- const turnNPlusOneMessages: ModelMessage[] = [- ...turnNMessages,- { role: 'assistant', content: 'Second answer' },- { role: 'user', content: 'Third question' }- ]-- const turnNResult = appendRelatedQuestionsReminder(turnNMessages)- const turnNPlusOneResult =- appendRelatedQuestionsReminder(turnNPlusOneMessages)-- expect(turnNResult.slice(0, turnNMessages.length)).toEqual(turnNMessages)- expect(turnNPlusOneResult.slice(0, turnNMessages.length)).toEqual(- turnNMessages- )- expect(turnNPlusOneResult[turnNMessages.length]).toEqual({- role: 'assistant',- content: 'Second answer'- })- })-- it('never leaves two consecutive user turns for a provider that requires alternating roles', () => {- const messages: ModelMessage[] = [- { role: 'user', content: 'First question' },- { role: 'assistant', content: 'First answer' },- { role: 'user', content: 'Follow-up question' }- ]-- const result = appendRelatedQuestionsReminder(messages, 'google')-- expect(result.map(message => message.role)).toEqual([- 'user',- 'assistant',- 'user'- ])- expect(result.at(-1)).toEqual({- role: 'user',- content: [- { type: 'text', text: 'Follow-up question' },- { type: 'text', text: RELATED_QUESTIONS_REMINDER }- ]- })- expect(messages.at(-1)).toEqual({- role: 'user',- content: 'Follow-up question'- })- })-- it('uses its own message for a provider that accepts consecutive user turns', () => {- const messages: ModelMessage[] = [- { role: 'user', content: 'Follow-up question' }- ]-- const result = appendRelatedQuestionsReminder(messages, 'openai')-- expect(result.at(-1)).toEqual({- role: 'user',- content: [{ type: 'text', text: RELATED_QUESTIONS_REMINDER }]- })- expect(result.at(-2)).toBe(messages.at(-1))- })-})diff --git a/lib/agents/prompts/related-questions-reminder.ts b/lib/agents/prompts/related-questions-reminder.tsdeleted file mode 100644index a9cc7692..00000000--- a/lib/agents/prompts/related-questions-reminder.ts+++ /dev/null@@ -1,53 +0,0 @@-import type { ModelMessage } from 'ai'--export const RELATED_QUESTIONS_REMINDER = `Response format reminder: When this turn produces a substantive answer, end it with exactly one related-questions \`\`\`spec JSONL block containing exactly three questions, using the schema in the system instructions. Omit the block only for the skip cases listed there.`--// Providers whose API rejects a prompt that does not strictly alternate roles,-// and so cannot be given the reminder as a message of its own.-const PROVIDERS_REQUIRING_ALTERNATING_ROLES = new Set(['google'])--/**- * Keeps the related-questions requirement next to the current request instead- * of leaving it only at the start of an increasingly long conversation.- *- * The reminder goes in a message of its own, because folding it into the- * current user message rewrites history that the next turn replays without it:- * the prompt prefix would then differ on every turn and stop being cacheable.- * Providers that require alternating roles cannot take the extra message, so- * they fall back to folding and keep the prefix cost.- */-export function appendRelatedQuestionsReminder(- messages: ModelMessage[],- providerId?: string-): ModelMessage[] {- const lastUserIndex = messages.findLastIndex(- message => message.role === 'user'- )-- if (lastUserIndex === -1) return messages-- const reminderPart = {- type: 'text' as const,- text: RELATED_QUESTIONS_REMINDER- }-- if (- providerId === undefined ||- !PROVIDERS_REQUIRING_ALTERNATING_ROLES.has(providerId)- ) {- return [...messages, { role: 'user', content: [reminderPart] }]- }-- const userMessage = messages[lastUserIndex]- if (userMessage.role !== 'user') return messages-- const content =- typeof userMessage.content === 'string'- ? [{ type: 'text' as const, text: userMessage.content }, reminderPart]- : [...userMessage.content, reminderPart]-- const updatedMessages = [...messages]- updatedMessages[lastUserIndex] = { ...userMessage, content }-- return updatedMessages-}diff --git a/lib/agents/researcher.ts b/lib/agents/researcher.tsindex f7e2f3fb..ef1f549f 100644--- a/lib/agents/researcher.ts+++ b/lib/agents/researcher.ts@@ -11,7 +11,6 @@ import { SearchMode } from '../types/search'import { getModel } from '../utils/registry'import { isTracingEnabled } from '../utils/telemetry'-import { appendRelatedQuestionsReminder } from './prompts/related-questions-reminder'import {getAdaptiveModePrompt,QUICK_MODE_PROMPT@@ -144,15 +143,6 @@ export function createResearcher({tools,activeTools: activeToolsList,stopWhen: stepCountIs(maxSteps),- prepareCall: options => ({- ...options,- ...(options.messages && {- messages: appendRelatedQuestionsReminder(- options.messages,- modelConfig?.providerId- )- })- }),...(providerOptions && { providerOptions }),// Spans join the parent Langfuse trace via OTel context propagationexperimental_telemetry: {diff --git a/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts b/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.tsnew file mode 100644index 00000000..4185a777--- /dev/null+++ b/lib/streaming/helpers/__tests__/strip-spec-from-messages.test.ts@@ -0,0 +1,95 @@+import type { UIMessage } from 'ai'+import { describe, expect, it } from 'vitest'++import { stripSpecFromMessages } from '../strip-spec-from-messages'++const RELATED_BLOCK = [+ '```spec',+ '{"op":"add","path":"/root","value":"main"}',+ '{"op":"add","path":"/elements/main","value":{"type":"Stack","props":{"direction":"vertical","gap":"sm"},"children":["header","questions"]}}',+ '{"op":"add","path":"/elements/header","value":{"type":"Heading","props":{"title":"Related","icon":"related"},"children":[]}}',+ '{"op":"add","path":"/elements/questions","value":{"type":"Stack","props":{"direction":"vertical","gap":"xs"},"children":["q1","q2","q3"]}}',+ '{"op":"add","path":"/elements/q1","value":{"type":"Button","props":{"text":"A follow-up question","variant":"link","icon":"arrow-right"},"on":{"press":{"action":"submitQuery","params":{"query":"A follow-up question"}}},"children":[]}}',+ '```'+].join('\n')++const IMAGE_BLOCK = [+ '```spec',+ '{"op":"add","path":"/root","value":"gallery"}',+ '{"op":"add","path":"/elements/gallery","value":{"type":"ImageGrid","props":{"columns":3},"children":[]}}',+ '```'+].join('\n')++function assistant(text: string): UIMessage {+ return {+ id: 'a1',+ role: 'assistant',+ parts: [{ type: 'text', text }]+ } as unknown as UIMessage+}++function textOf(message: UIMessage): string {+ const part = message.parts[0]+ return part.type === 'text' ? part.text : ''+}++describe('stripSpecFromMessages', () => {+ it('keeps the related-questions block', () => {+ const input = `Here is the answer.\n\n${RELATED_BLOCK}`+ const [result] = stripSpecFromMessages([assistant(input)])++ expect(textOf(result)).toBe(input)+ })++ it('strips a spec block that is not the related-questions block', () => {+ const input = `Here is the answer.\n\n${IMAGE_BLOCK}`+ const [result] = stripSpecFromMessages([assistant(input)])++ expect(textOf(result)).toBe('Here is the answer.')+ })++ it('keeps only the related block when a message carries both', () => {+ const input = [+ 'Intro paragraph.',+ '',+ IMAGE_BLOCK,+ '',+ 'Closing paragraph.',+ '',+ RELATED_BLOCK+ ].join('\n')++ const [result] = stripSpecFromMessages([assistant(input)])++ expect(textOf(result)).toBe(+ ['Intro paragraph.', '', 'Closing paragraph.', '', RELATED_BLOCK].join(+ '\n'+ )+ )+ })++ it('passes text without a spec block through byte-identically', () => {+ const input = '# Title\n\n\nBody with trailing space \n\nand a tail.\n\n'+ const [result] = stripSpecFromMessages([assistant(input)])++ expect(textOf(result)).toBe(input)+ })++ it('leaves user and system messages untouched', () => {+ const user = {+ id: 'u1',+ role: 'user',+ parts: [{ type: 'text', text: `Question\n\n${IMAGE_BLOCK}` }]+ } as unknown as UIMessage+ const system = {+ id: 's1',+ role: 'system',+ parts: [{ type: 'text', text: `Instructions\n\n${IMAGE_BLOCK}` }]+ } as unknown as UIMessage++ const result = stripSpecFromMessages([user, system])++ expect(result[0]).toBe(user)+ expect(result[1]).toBe(system)+ })+})diff --git a/lib/streaming/helpers/strip-spec-from-messages.ts b/lib/streaming/helpers/strip-spec-from-messages.tsindex bfe4e5c9..423c5088 100644--- a/lib/streaming/helpers/strip-spec-from-messages.ts+++ b/lib/streaming/helpers/strip-spec-from-messages.ts@@ -1,12 +1,45 @@import { UIMessage } from 'ai'-import { stripSpecBlocks } from '@/lib/render/strip-spec-blocks'+const SPEC_FENCE = /```spec[\s\S]*?```/g++// The related-questions block is the only spec block whose JSONL carries a+// heading with this icon.+const RELATED_QUESTIONS_MARKER = '"icon":"related"'++/**+ * Removes ```spec fenced blocks from assistant message text, keeping the+ * related-questions block. Dropping the rest stops prior spec payloads from+ * spending context window budget on every later turn; keeping the related one+ * leaves the model an example of the format it is asked to keep producing.+ *+ * The result must depend only on the text passed in, because this runs on the+ * prompt-cache prefix and a message has to replay identically on every turn.+ * That is also why this does not reuse the clipboard helper, whose job is+ * readable output rather than a stable prefix.+ */+function stripNonRelatedSpecBlocks(text: string): string {+ let removed = false++ const stripped = text.replace(SPEC_FENCE, block => {+ if (block.includes(RELATED_QUESTIONS_MARKER)) {+ return block+ }+ removed = true+ return ''+ })++ if (!removed) {+ return text+ }++ // Only normalize when a block was taken out, so that text without spec+ // blocks passes through unchanged.+ return stripped.replace(/\n{3,}/g, '\n\n').trim()+}/**- * Strips ```spec fenced blocks from assistant message text parts- * before converting to model messages. This prevents prior spec- * payloads from being fed back into the model on subsequent turns,- * wasting context window budget.+ * Applies the strip to assistant message text parts before converting to+ * model messages.*/export function stripSpecFromMessages(messages: UIMessage[]): UIMessage[] {return messages.map(msg => {@@ -16,7 +49,7 @@ export function stripSpecFromMessages(messages: UIMessage[]): UIMessage[] {const parts = msg.parts.map(part => {if (part.type === 'text' && typeof part.text === 'string') {- const stripped = stripSpecBlocks(part.text)+ const stripped = stripNonRelatedSpecBlocks(part.text)if (stripped !== part.text) {return { ...part, text: stripped }}