Skip to content

An AI-powered search engine with a generative UI

fix(openai): set per-chat prompt cache key (#959)

Sushant Tiwari73d041c708a6
diff --git a/lib/agents/__tests__/researcher.test.ts b/lib/agents/__tests__/researcher.test.ts
new file mode 100644
index 00000000..5e2c5981
--- /dev/null
+++ b/lib/agents/__tests__/researcher.test.ts
@@ -0,0 +1,117 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+import type { Model } from '@/lib/types/models'
+
+const mocks = vi.hoisted(() => ({
+ agentOptions: undefined as Record<string, any> | undefined
+}))
+
+vi.mock('ai', () => ({
+ stepCountIs: vi.fn(),
+ tool: vi.fn(options => options),
+ ToolLoopAgent: vi.fn(function (
+ this: { tools: unknown },
+ options: Record<string, any>
+ ) {
+ mocks.agentOptions = options
+ this.tools = options.tools
+ })
+}))
+
+vi.mock('@/lib/tools/fetch', () => ({
+ fetchTool: {}
+}))
+
+vi.mock('@/lib/tools/question', () => ({
+ createQuestionTool: vi.fn(() => ({}))
+}))
+
+vi.mock('@/lib/tools/search', () => ({
+ createSearchTool: vi.fn(() => ({}))
+}))
+
+vi.mock('@/lib/tools/todo', () => ({
+ createTodoTools: vi.fn(() => ({ todoWrite: {} }))
+}))
+
+vi.mock('@/lib/utils/registry', () => ({
+ getModel: vi.fn(() => ({}))
+}))
+
+vi.mock('@/lib/utils/telemetry', () => ({
+ isTracingEnabled: vi.fn(() => false)
+}))
+
+vi.mock('../prompts/search-mode-prompts', () => ({
+ getAdaptiveModePrompt: vi.fn(() => 'Adaptive prompt'),
+ QUICK_MODE_PROMPT: 'Quick prompt'
+}))
+
+import { createResearcher } from '../researcher'
+
+function createModel(
+ providerId: string,
+ providerOptions?: Model['providerOptions']
+): Model {
+ return {
+ id: 'model-id',
+ name: 'Model',
+ provider: 'Provider',
+ providerId,
+ providerOptions
+ }
+}
+
+describe('createResearcher', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ mocks.agentOptions = undefined
+ vi.spyOn(console, 'log').mockImplementation(() => {})
+ })
+
+ it('adds the chat ID to existing OpenAI provider options', () => {
+ createResearcher({
+ model: 'openai:model-id',
+ modelConfig: createModel('openai', {
+ openai: { reasoningEffort: 'low' },
+ custom: { enabled: true }
+ }),
+ chatId: 'chat-id'
+ })
+
+ expect(mocks.agentOptions?.providerOptions).toEqual({
+ openai: {
+ reasoningEffort: 'low',
+ promptCacheKey: 'chat-id'
+ },
+ custom: { enabled: true }
+ })
+ })
+
+ it('does not add OpenAI options for another provider', () => {
+ createResearcher({
+ model: 'google:model-id',
+ modelConfig: createModel('google', {
+ google: { thinkingConfig: { thinkingBudget: 1024 } }
+ }),
+ chatId: 'chat-id'
+ })
+
+ expect(mocks.agentOptions?.providerOptions).toEqual({
+ google: { thinkingConfig: { thinkingBudget: 1024 } }
+ })
+ })
+
+ it('preserves OpenAI options when no chat ID is available', () => {
+ createResearcher({
+ model: 'openai:model-id',
+ modelConfig: createModel('openai', {
+ openai: { reasoningEffort: 'low' }
+ })
+ })
+
+ expect(mocks.agentOptions?.providerOptions).toEqual({
+ openai: { reasoningEffort: 'low' }
+ })
+ })
+})
diff --git a/lib/agents/researcher.ts b/lib/agents/researcher.ts
index 722b6d22..ef1f549f 100644
--- a/lib/agents/researcher.ts
+++ b/lib/agents/researcher.ts
@@ -70,10 +70,12 @@ function wrapSearchToolForQuickMode<
export function createResearcher({
model,
modelConfig,
+ chatId,
searchMode = 'adaptive'
}: {
model: string
modelConfig?: Model
+ chatId?: string
searchMode?: SearchMode
}) {
try {
@@ -123,6 +125,17 @@ export function createResearcher({
...todoTools
} as ResearcherTools
+ const providerOptions =
+ modelConfig?.providerId === 'openai' && chatId
+ ? {
+ ...modelConfig.providerOptions,
+ openai: {
+ ...modelConfig.providerOptions?.openai,
+ promptCacheKey: chatId
+ }
+ }
+ : modelConfig?.providerOptions
+
// Create ToolLoopAgent with all configuration
const agent = new ToolLoopAgent({
model: getModel(model),
@@ -130,9 +143,7 @@ export function createResearcher({
tools,
activeTools: activeToolsList,
stopWhen: stepCountIs(maxSteps),
- ...(modelConfig?.providerOptions && {
- providerOptions: modelConfig.providerOptions
- }),
+ ...(providerOptions && { providerOptions }),
// Spans join the parent Langfuse trace via OTel context propagation
experimental_telemetry: {
isEnabled: isTracingEnabled(),
diff --git a/lib/streaming/__tests__/create-chat-stream-response.test.ts b/lib/streaming/__tests__/create-chat-stream-response.test.ts
index 4220d07a..6cbcc5a5 100644
--- a/lib/streaming/__tests__/create-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-chat-stream-response.test.ts
@@ -66,6 +66,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({
logUsage: vi.fn()
}))
+import { researcher } from '@/lib/agents/researcher'
import { createChatStreamResponse } from '@/lib/streaming/create-chat-stream-response'
import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'
@@ -133,6 +134,17 @@ describe('createChatStreamResponse', () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
})
+ it('passes the chat ID to the researcher', async () => {
+ mocks.stream.mockResolvedValue(createFakeResult())
+
+ await createChatStreamResponse(createConfig())
+ await mocks.finishPromise
+
+ expect(researcher).toHaveBeenCalledWith(
+ expect.objectContaining({ chatId: 'chat-id' })
+ )
+ })
+
it('does not mark the span as failed for an aborted stream error', async () => {
const streamError = new Error('request stopped')
streamError.name = 'ResponseAborted'
diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
index d79cf289..092729c0 100644
--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
@@ -47,6 +47,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({
logUsage: vi.fn()
}))
+import { researcher } from '@/lib/agents/researcher'
import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'
import { createEphemeralChatStreamResponse } from '@/lib/streaming/create-ephemeral-chat-stream-response'
import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'
@@ -115,6 +116,7 @@ function createConfig(abortSignal: AbortSignal = new AbortController().signal) {
}
],
model: { providerId: 'openai', id: 'gpt-4o-mini' } as any,
+ chatId: 'chat-id',
abortSignal,
searchMode: 'quick' as const
}
@@ -128,6 +130,17 @@ describe('createEphemeralChatStreamResponse', () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
})
+ it('passes the chat ID to the researcher', async () => {
+ mocks.stream.mockResolvedValue(createFakeResult())
+
+ await createEphemeralChatStreamResponse(createConfig())
+ await mocks.finishPromise
+
+ expect(researcher).toHaveBeenCalledWith(
+ expect.objectContaining({ chatId: 'chat-id' })
+ )
+ })
+
it('returns 400 when messages are missing', async () => {
const response = await createEphemeralChatStreamResponse({
...createConfig(),
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index cd046eb0..b28a559f 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -150,6 +150,7 @@ export async function createChatStreamResponse(
const researchAgent = researcher({
model: context.modelId,
modelConfig: model,
+ chatId,
searchMode
})
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 6d5152a5..6ba9c2e0 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -107,6 +107,7 @@ export async function createEphemeralChatStreamResponse(
const researchAgent = researcher({
model: `${model.providerId}:${model.id}`,
modelConfig: model,
+ chatId,
searchMode
})