An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(streaming): keep a recovered tool failure off the turn's root span (#1031)
Yoshiki Miura1ebad188eed9
diff --git a/lib/streaming/__tests__/create-chat-stream-response.test.ts b/lib/streaming/__tests__/create-chat-stream-response.test.tsindex 2e64df96..4aed00c9 100644--- a/lib/streaming/__tests__/create-chat-stream-response.test.ts+++ b/lib/streaming/__tests__/create-chat-stream-response.test.ts@@ -76,6 +76,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({import { loadChat, loadChatUncached } from '@/lib/actions/chat'import { researcher } from '@/lib/agents/researcher'import { DeterministicPreparationError } from '@/lib/errors/deterministic-preparation-error'+import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'import { createChatStreamResponse } from '@/lib/streaming/create-chat-stream-response'import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'import { EMPTY_RESPONSE_STATUS_MESSAGE } from '@/lib/streaming/helpers/is-empty-response'@@ -91,22 +92,30 @@ type UIMessageStreamResponseOptions = {responseMessage: {id: stringrole: 'assistant'- parts: Array<{ type: string; text?: string }>+ parts: Array<{ type: string; text?: string; state?: string }>}isAborted: boolean}) => Promise<void>+ onError: (error: unknown) => string}function createFakeResult(isAborted = false,- parts: Array<{ type: string; text?: string }> = [+ parts: Array<{ type: string; text?: string; state?: string }> = [{ type: 'text', text: 'Answer' }- ]+ ],+ // Raised into the response handler before the turn ends, the way a tool+ // failure reaches it while the stream keeps going.+ responseError?: { error: unknown; onSerialized?: (payload: string) => void }) {return {consumeStream: vi.fn(),toUIMessageStreamResponse: vi.fn((options: UIMessageStreamResponseOptions) => {+ if (responseError) {+ const serialized = options.onError(responseError.error)+ if (responseError.onSerialized) responseError.onSerialized(serialized)+ }mocks.finishPromise = options.onEnd({responseMessage: {id: 'response-id',@@ -595,4 +604,81 @@ describe('createChatStreamResponse', () => {expect(onZeroPartError).toHaveBeenCalledOnce()})++ it('keeps the turn successful when a tool fails and the answer still lands', async () => {+ const onZeroPartError = vi.fn(async () => undefined)+ const toolFailure = new ToolFailureError(+ 'fetch',+ new Error('HTTP 403: Forbidden')+ )+ let serialized: string | undefined+ mocks.stream.mockResolvedValue(+ createFakeResult(+ false,+ [+ { type: 'tool-fetch', state: 'output-error' },+ { type: 'text', text: 'Answer' }+ ],+ {+ error: toolFailure,+ onSerialized: payload => {+ serialized = payload+ }+ }+ )+ )++ await createChatStreamResponse({ ...createConfig(), onZeroPartError })+ await mocks.finishPromise++ expect(serialized).toBe(serializeToolFailure(toolFailure))+ expect(mocks.span.update).toHaveBeenCalledWith({+ input: 'hello',+ output: 'Answer'+ })+ expect(onZeroPartError).not.toHaveBeenCalled()+ })++ it('marks the turn as failed when a tool fails and nothing is answered', async () => {+ const onZeroPartError = vi.fn(async () => undefined)+ const toolFailure = new ToolFailureError(+ 'search',+ new Error('Tavily search failed: HTTP 400: Bad Request')+ )+ mocks.stream.mockResolvedValue(+ createFakeResult(false, [], { error: toolFailure })+ )++ await createChatStreamResponse({ ...createConfig(), onZeroPartError })+ await mocks.finishPromise++ const update = mocks.span.update.mock.calls.at(-1)?.[0]+ expect(update).toMatchObject({ input: 'hello', level: 'ERROR' })+ expect(update).not.toHaveProperty('output')+ expect(onZeroPartError).toHaveBeenCalledOnce()+ })++ it('marks the turn as failed when a tool failure leaves only a preamble', async () => {+ const toolFailure = new ToolFailureError(+ 'fetch',+ new Error('HTTP 403: Forbidden')+ )+ mocks.stream.mockResolvedValue(+ createFakeResult(+ false,+ [+ { type: 'text', text: 'Let me look that up.' },+ { type: 'tool-fetch', state: 'output-error' }+ ],+ { error: toolFailure }+ )+ )++ await createChatStreamResponse(createConfig())+ await mocks.finishPromise++ const update = mocks.span.update.mock.calls.at(-1)?.[0]+ expect(update).toMatchObject({ input: 'hello', level: 'ERROR' })+ expect(update).not.toHaveProperty('output')+ })})diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 6dd91bea..0b215010 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -34,6 +34,7 @@ import { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'import { dedupeAttachments } from './helpers/dedupe-attachments'import { describeTurnInput } from './helpers/describe-turn-input'+import { hasAnswerAfterToolFailure } from './helpers/has-answer-after-tool-failure'import { hasResponseContentPart } from './helpers/has-response-content'import {EMPTY_RESPONSE_STATUS_MESSAGE,@@ -119,6 +120,10 @@ export async function createChatStreamResponse(// be attached to this trace laterconst parentTraceId = rootSpan?.traceIdlet hasStreamError = false+ // A failing tool is the page or the search service failing, not the turn:+ // the agent keeps going and the answer still lands. Tracked apart from the+ // stream's own failures so it can only become one when nothing was answered.+ let hasToolFailure = falselet hasEmptyResponse = falselet streamError: unknown// The agent stream call is preparation until its first generation starts.@@ -296,13 +301,18 @@ export async function createChatStreamResponse(`researchAgent.stream - Start: model=${context.modelId}, searchMode=${searchMode}`)streamErrorStage = 'start-stream'- // AgentStreamParameters omits onError, but it reaches streamText where only- // stream errors, not recoverable tool errors, invoke it.+ // AgentStreamParameters omits onError, but it reaches streamText. A tool+ // failure is classified here too rather than assumed away: the same error+ // is recoverable whichever handler it arrives at.const result = await researchAgent.stream({messages: modelMessages,abortSignal,onError: ({ error }) => {- hasStreamError = true+ if (isToolFailureError(error)) {+ hasToolFailure = true+ } else {+ hasStreamError = true+ }streamError = errorstreamErrorWasCancelled = abortSignal?.aborted ?? falsestreamErrorPhase = 'generation'@@ -347,12 +357,22 @@ export async function createChatStreamResponse(perfTime('researchAgent.stream completed', llmStart)if (isAborted) returnif (!responseMessage) {- if (hasStreamError) await onZeroPartError?.()+ if (hasStreamError || hasToolFailure) {+ hasStreamError = true+ await onZeroPartError?.()+ }return}rootOutput = getTextFromParts(responseMessage.parts) || undefinedhasEmptyResponse = isEmptyResponse(responseMessage)+ // The tool failure is only the turn's failure once the turn has no+ // answer to show for it, and an answer counts only when it came+ // after the failure: text the model wrote on its way to the tool is+ // a preamble.+ if (hasToolFailure && !hasAnswerAfterToolFailure(responseMessage)) {+ hasStreamError = true+ }if (hasEmptyResponse ||(hasStreamError && !hasResponseContentPart(responseMessage))@@ -377,16 +397,17 @@ export async function createChatStreamResponse(}},onError: (error: unknown) => {- hasStreamError = truestreamError = errorstreamErrorWasCancelled = abortSignal?.aborted ?? falsestreamErrorPhase = 'generation'if (isToolFailureError(error)) {+ hasToolFailure = trueconsole.error('Tool failure:', error)return serializeToolFailure(error)}+ hasStreamError = truelogAPICallErrorDiagnostics(error)console.error('Stream response error:', error)return serializePublicError(error)diff --git a/lib/streaming/helpers/__tests__/has-answer-after-tool-failure.test.ts b/lib/streaming/helpers/__tests__/has-answer-after-tool-failure.test.tsnew file mode 100644index 00000000..fe23393f--- /dev/null+++ b/lib/streaming/helpers/__tests__/has-answer-after-tool-failure.test.ts@@ -0,0 +1,58 @@+import { describe, expect, it } from 'vitest'++import { hasAnswerAfterToolFailure } from '../has-answer-after-tool-failure'++describe('hasAnswerAfterToolFailure', () => {+ it('accepts answer text written after the failed call', () => {+ expect(+ hasAnswerAfterToolFailure({+ parts: [+ { type: 'tool-fetch', state: 'output-available' },+ { type: 'tool-fetch', state: 'output-error' },+ { type: 'text', text: 'Answer' }+ ]+ })+ ).toBe(true)+ })++ it('rejects text that only preceded the failed call', () => {+ expect(+ hasAnswerAfterToolFailure({+ parts: [+ { type: 'text', text: 'Let me look that up.' },+ { type: 'tool-fetch', state: 'output-error' }+ ]+ })+ ).toBe(false)+ })++ it('rejects whitespace standing in for an answer', () => {+ expect(+ hasAnswerAfterToolFailure({+ parts: [+ { type: 'tool-search', state: 'output-error' },+ { type: 'text', text: ' \n' }+ ]+ })+ ).toBe(false)+ })++ it('reads the last failure, not the first', () => {+ expect(+ hasAnswerAfterToolFailure({+ parts: [+ { type: 'tool-fetch', state: 'output-error' },+ { type: 'text', text: 'Partial' },+ { type: 'tool-fetch', state: 'output-error' }+ ]+ })+ ).toBe(false)+ })++ it('rejects a message that carries no failed call at all', () => {+ expect(+ hasAnswerAfterToolFailure({ parts: [{ type: 'text', text: 'Answer' }] })+ ).toBe(false)+ expect(hasAnswerAfterToolFailure(undefined)).toBe(false)+ })+})diff --git a/lib/streaming/helpers/has-answer-after-tool-failure.ts b/lib/streaming/helpers/has-answer-after-tool-failure.tsnew file mode 100644index 00000000..57b3a0d2--- /dev/null+++ b/lib/streaming/helpers/has-answer-after-tool-failure.ts@@ -0,0 +1,37 @@+type ResponsePart = {+ type?: string+ state?: string+ text?: string+}++type ResponseWithParts = {+ parts?: ResponsePart[]+}++/**+ * Whether the turn answered after its last failed tool call.+ *+ * A tool failure the agent recovered from is followed by the answer it wrote+ * with whatever the other calls returned. A tool failure that ended the turn+ * leaves only what the model had said before reaching for the tool, which is a+ * preamble and not an answer. Both messages carry text, so position is what+ * separates them.+ *+ * A failure that never reached the message cannot be placed, so the turn is not+ * credited with an answer it cannot be shown to have produced.+ */+export function hasAnswerAfterToolFailure(+ response: ResponseWithParts | null | undefined+): boolean {+ const parts = response?.parts ?? []++ let lastFailure = -1+ for (let index = 0; index < parts.length; index += 1) {+ if (parts[index]?.state === 'output-error') lastFailure = index+ }+ if (lastFailure === -1) return false++ return parts+ .slice(lastFailure + 1)+ .some(part => part.type === 'text' && (part.text ?? '').trim().length > 0)+}