An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(tracing): distinguish unclassified stream failures from one another and from user aborts (#956)
Yoshiki Miura6c051a99bd93
diff --git a/lib/streaming/__tests__/create-chat-stream-response.test.ts b/lib/streaming/__tests__/create-chat-stream-response.test.tsnew file mode 100644index 00000000..4220d07a--- /dev/null+++ b/lib/streaming/__tests__/create-chat-stream-response.test.ts@@ -0,0 +1,199 @@+import { beforeEach, describe, expect, it, vi } from 'vitest'++const mocks = vi.hoisted(() => ({+ stream: vi.fn(),+ span: {+ traceId: 'trace-id',+ update: vi.fn(),+ end: vi.fn()+ },+ forceFlush: vi.fn(),+ finishPromise: Promise.resolve()+}))++vi.mock('ai', () => ({+ consumeStream: vi.fn(),+ convertToModelMessages: vi.fn(async () => []),+ smoothStream: vi.fn()+}))++vi.mock('@langfuse/tracing', () => ({+ propagateAttributes: vi.fn((_attributes: unknown, callback: () => unknown) =>+ callback()+ ),+ startActiveObservation: vi.fn(+ (_name: string, callback: (span: typeof mocks.span) => unknown) =>+ callback(mocks.span)+ )+}))++vi.mock('@/instrumentation', () => ({+ langfuseSpanProcessor: {+ forceFlush: mocks.forceFlush+ }+}))++vi.mock('@/lib/actions/chat', () => ({+ loadChat: vi.fn()+}))++vi.mock('@/lib/agents/researcher', () => ({+ researcher: vi.fn(() => ({ stream: mocks.stream }))+}))++vi.mock('@/lib/agents/title-generator', () => ({+ generateChatTitle: vi.fn(async () => 'Title')+}))++vi.mock('@/lib/streaming/helpers/attachment-sizes', () => ({+ resolveAttachmentSizes: vi.fn(async (messages: unknown) => messages)+}))++vi.mock('@/lib/streaming/helpers/persist-stream-results', () => ({+ persistStreamResults: vi.fn(async () => undefined)+}))++vi.mock('@/lib/streaming/helpers/prepare-messages', () => ({+ prepareMessages: vi.fn(async () => [])+}))++vi.mock('@/lib/utils/telemetry', () => ({+ isTracingEnabled: vi.fn(() => true)+}))++vi.mock('@/lib/utils/usage-logging', () => ({+ isUsageLogging: vi.fn(() => false),+ logUsage: vi.fn()+}))++import { createChatStreamResponse } from '@/lib/streaming/create-chat-stream-response'+import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'++type StreamOptions = {+ onError: (event: { error: unknown }) => void+}++type UIMessageStreamResponseOptions = {+ onFinish: (event: {+ responseMessage: {+ id: string+ role: 'assistant'+ parts: Array<{ type: string; text?: string }>+ }+ isAborted: boolean+ }) => Promise<void>+}++function createFakeResult(isAborted = false) {+ return {+ consumeStream: vi.fn(),+ toUIMessageStreamResponse: vi.fn(+ (options: UIMessageStreamResponseOptions) => {+ mocks.finishPromise = options.onFinish({+ responseMessage: {+ id: 'response-id',+ role: 'assistant',+ parts: [{ type: 'text', text: 'Answer' }]+ },+ isAborted+ })+ return new Response()+ }+ )+ }+}++function createAbortedSignal(): AbortSignal {+ const controller = new AbortController()+ controller.abort()++ return controller.signal+}++function createConfig(abortSignal: AbortSignal = new AbortController().signal) {+ return {+ message: {+ id: 'message-id',+ role: 'user' as const,+ parts: [{ type: 'text' as const, text: 'hello' }]+ },+ model: { providerId: 'openai', id: 'gpt-4o-mini' } as any,+ chatId: 'chat-id',+ userId: 'user-id',+ abortSignal,+ isNewChat: true,+ searchMode: 'quick' as const+ }+}++describe('createChatStreamResponse', () => {+ beforeEach(() => {+ vi.clearAllMocks()+ mocks.finishPromise = Promise.resolve()+ vi.spyOn(console, 'error').mockImplementation(() => {})+ })++ it('does not mark the span as failed for an aborted stream error', async () => {+ const streamError = new Error('request stopped')+ streamError.name = 'ResponseAborted'+ mocks.stream.mockImplementation(async (options: StreamOptions) => {+ options.onError({ error: streamError })+ return createFakeResult(true)+ })++ await createChatStreamResponse(createConfig(createAbortedSignal()))+ await mocks.finishPromise++ expect(mocks.span.update).not.toHaveBeenCalled()+ expect(mocks.span.end).toHaveBeenCalledOnce()+ expect(mocks.forceFlush).toHaveBeenCalledOnce()+ })++ it('marks the span as failed when the client disconnects after the failure', async () => {+ const streamError = new Error('upstream stopped')+ streamError.name = 'AbortError'+ const controller = new AbortController()+ mocks.stream.mockImplementation(async (options: StreamOptions) => {+ options.onError({ error: streamError })+ controller.abort()+ return createFakeResult()+ })++ await createChatStreamResponse(createConfig(controller.signal))+ await mocks.finishPromise++ expect(mocks.span.update).toHaveBeenCalledWith({+ level: 'ERROR',+ statusMessage: describeStreamError(streamError),+ metadata: { streamErrorShape: { name: 'AbortError' } }+ })+ })++ it('attaches shape metadata for an unclassified stream error', async () => {+ const streamError = Object.assign(new Error('private failure detail'), {+ name: 'StreamFailure',+ errno: 91+ })+ mocks.stream.mockImplementation(async (options: StreamOptions) => {+ options.onError({ error: streamError })+ return createFakeResult()+ })++ await createChatStreamResponse(createConfig())+ await mocks.finishPromise++ expect(mocks.span.update).toHaveBeenCalledWith({+ level: 'ERROR',+ statusMessage: describeStreamError(streamError),+ metadata: {+ streamErrorShape: {+ name: 'StreamFailure',+ errno: 91+ }+ }+ })+ expect(JSON.stringify(mocks.span.update.mock.calls)).not.toContain(+ 'private failure detail'+ )+ })+})diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.tsindex 3788ad14..d79cf289 100644--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts@@ -98,7 +98,14 @@ function createFakeResult({}}-function createConfig() {+function createAbortedSignal(): AbortSignal {+ const controller = new AbortController()+ controller.abort()++ return controller.signal+}++function createConfig(abortSignal: AbortSignal = new AbortController().signal) {return {messages: [{@@ -108,7 +115,7 @@ function createConfig() {}],model: { providerId: 'openai', id: 'gpt-4o-mini' } as any,- abortSignal: new AbortController().signal,+ abortSignal,searchMode: 'quick' as const}}@@ -148,8 +155,11 @@ describe('createEphemeralChatStreamResponse', () => {expect(mocks.forceFlush).toHaveBeenCalledOnce()})- it('marks the span as failed for a stream-level error', async () => {- const streamError = new Error('stream failed')+ it('attaches shape metadata for an unclassified stream error', async () => {+ const streamError = Object.assign(new Error('stream failed'), {+ name: 'StreamFailure',+ code: 'STREAM_CODE'+ })mocks.stream.mockImplementation(async (options: StreamOptions) => {options.onError({ error: streamError })return createFakeResult()@@ -160,12 +170,34 @@ describe('createEphemeralChatStreamResponse', () => {expect(mocks.span.update).toHaveBeenCalledWith({level: 'ERROR',- statusMessage: describeStreamError(streamError)+ statusMessage: describeStreamError(streamError),+ metadata: {+ streamErrorShape: {+ name: 'StreamFailure',+ code: 'STREAM_CODE'+ }+ }})expect(mocks.span.end).toHaveBeenCalledOnce()expect(mocks.forceFlush).toHaveBeenCalledOnce()})+ it('does not mark the span as failed for an aborted stream error', async () => {+ const streamError = new Error('request stopped')+ streamError.name = 'AbortError'+ mocks.stream.mockImplementation(async (options: StreamOptions) => {+ options.onError({ error: streamError })+ return createFakeResult({ isAborted: true })+ })++ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))+ await mocks.finishPromise++ expect(mocks.span.update).not.toHaveBeenCalled()+ expect(mocks.span.end).toHaveBeenCalledOnce()+ expect(mocks.forceFlush).toHaveBeenCalledOnce()+ })+it('marks the span as failed for an empty response', async () => {mocks.stream.mockResolvedValue(createFakeResult({ parts: [] }))@@ -181,7 +213,7 @@ describe('createEphemeralChatStreamResponse', () => {it('does not update the span for a response with text', async () => {mocks.stream.mockResolvedValue(createFakeResult())- await createEphemeralChatStreamResponse(createConfig())+ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))await mocks.finishPromiseexpect(mocks.span.update).not.toHaveBeenCalled()@@ -192,7 +224,7 @@ describe('createEphemeralChatStreamResponse', () => {createFakeResult({ parts: [], isAborted: true }))- await createEphemeralChatStreamResponse(createConfig())+ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))await mocks.finishPromiseexpect(mocks.span.update).not.toHaveBeenCalled()@@ -211,7 +243,10 @@ describe('createEphemeralChatStreamResponse', () => {expect(mocks.span.update).toHaveBeenCalledOnce()expect(mocks.span.update).toHaveBeenCalledWith({level: 'ERROR',- statusMessage: describeStreamError(streamError)+ statusMessage: describeStreamError(streamError),+ metadata: {+ streamErrorShape: { name: 'Error' }+ }})})})diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 5c3005bb..cd046eb0 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -31,17 +31,14 @@ import { compactHistoricalMessages } from './helpers/compact-historical-messagesimport { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'import { dedupeAttachments } from './helpers/dedupe-attachments'-import { describeStreamError } from './helpers/describe-stream-error'import {EMPTY_RESPONSE_STATUS_MESSAGE,isEmptyResponse} from './helpers/is-empty-response'-import {- buildAPICallErrorDiagnostics,- logAPICallErrorDiagnostics-} from './helpers/log-api-call-error'+import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'import { persistStreamResults } from './helpers/persist-stream-results'import { prepareMessages } from './helpers/prepare-messages'+import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'import type { StreamContext } from './helpers/types'import { BaseStreamConfig } from './types'@@ -100,16 +97,19 @@ export async function createChatStreamResponse(let hasStreamError = falselet hasEmptyResponse = falselet streamError: unknown+ // Sampled where the failure happens: the client can disconnect before the+ // span is closed, and by then the signal no longer says whether the user+ // cancelled this turn or the failure was the model's own.+ let streamErrorWasCancelled = falseconst endTracing = async () => {if (rootSpan) {if (hasStreamError) {- const apiCallDiagnostics = buildAPICallErrorDiagnostics(streamError)- rootSpan.update({- level: 'ERROR',- statusMessage: describeStreamError(streamError),- ...(apiCallDiagnostics && { metadata: { apiCallDiagnostics } })- })+ const update = buildStreamErrorSpanUpdate(+ streamError,+ streamErrorWasCancelled+ )+ if (update) rootSpan.update(update)} else if (hasEmptyResponse) {rootSpan.update({level: 'ERROR',@@ -207,6 +207,7 @@ export async function createChatStreamResponse(onError: ({ error }) => {hasStreamError = truestreamError = error+ streamErrorWasCancelled = abortSignal?.aborted ?? false},experimental_transform: smoothStream({ chunking: 'word' }),...(isUsageLogging() && {@@ -281,6 +282,7 @@ export async function createChatStreamResponse(} catch (error) {hasStreamError = truestreamError = error+ streamErrorWasCancelled = abortSignal?.aborted ?? falseawait endTracing()logAPICallErrorDiagnostics(error)console.error('Stream execution error:', error)diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex 9d90e527..6d5152a5 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -26,15 +26,12 @@ import { compactHistoricalMessages } from './helpers/compact-historical-messagesimport { convertDataPart } from './helpers/convert-data-part'import { assignDataPartNonces } from './helpers/data-part-nonce'import { dedupeAttachments } from './helpers/dedupe-attachments'-import { describeStreamError } from './helpers/describe-stream-error'import {EMPTY_RESPONSE_STATUS_MESSAGE,isEmptyResponse} from './helpers/is-empty-response'-import {- buildAPICallErrorDiagnostics,- logAPICallErrorDiagnostics-} from './helpers/log-api-call-error'+import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'+import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'import { BaseStreamConfig } from './types'@@ -67,16 +64,19 @@ export async function createEphemeralChatStreamResponse(let hasStreamError = falselet hasEmptyResponse = falselet streamError: unknown+ // Sampled where the failure happens: the client can disconnect before the+ // span is closed, and by then the signal no longer says whether the user+ // cancelled this turn or the failure was the model's own.+ let streamErrorWasCancelled = falseconst endTracing = async () => {if (rootSpan) {if (hasStreamError) {- const apiCallDiagnostics = buildAPICallErrorDiagnostics(streamError)- rootSpan.update({- level: 'ERROR',- statusMessage: describeStreamError(streamError),- ...(apiCallDiagnostics && { metadata: { apiCallDiagnostics } })- })+ const update = buildStreamErrorSpanUpdate(+ streamError,+ streamErrorWasCancelled+ )+ if (update) rootSpan.update(update)} else if (hasEmptyResponse) {rootSpan.update({level: 'ERROR',@@ -119,6 +119,7 @@ export async function createEphemeralChatStreamResponse(onError: ({ error }) => {hasStreamError = truestreamError = error+ streamErrorWasCancelled = abortSignal?.aborted ?? false},experimental_transform: smoothStream({ chunking: 'word' }),...(isUsageLogging() && {@@ -172,6 +173,7 @@ export async function createEphemeralChatStreamResponse(} catch (error) {hasStreamError = truestreamError = error+ streamErrorWasCancelled = abortSignal?.aborted ?? falseawait endTracing()logAPICallErrorDiagnostics(error)console.error('Ephemeral stream execution error:', error)diff --git a/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.tsnew file mode 100644index 00000000..eff90160--- /dev/null+++ b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts@@ -0,0 +1,111 @@+import { describe, expect, it } from 'vitest'++import {+ buildStreamErrorShape,+ buildStreamErrorSpanUpdate,+ isStreamAbortError+} from '../stream-error-diagnostics'++describe('isStreamAbortError', () => {+ it.each(['AbortError', 'ResponseAborted'])('recognizes %s errors', name => {+ const error = new Error('request stopped')+ error.name = name++ expect(isStreamAbortError(error)).toBe(true)+ })++ it('recognizes an abort DOMException', () => {+ expect(+ isStreamAbortError(new DOMException('request stopped', 'AbortError'))+ ).toBe(true)+ })++ it('recognizes an abort error in the cause', () => {+ const cause = new Error('request stopped')+ cause.name = 'ResponseAborted'++ expect(isStreamAbortError(new Error('wrapper', { cause }))).toBe(true)+ })++ it.each([+ new Error('AbortError'),+ Object.assign(new Error('tool timed out'), { name: 'TimeoutError' }),+ { name: 'AbortError' },+ new Error('ordinary failure')+ ])('does not recognize non-abort errors', error => {+ expect(isStreamAbortError(error)).toBe(false)+ })+})++describe('buildStreamErrorShape', () => {+ it('returns null for a classified error', () => {+ expect(buildStreamErrorShape(new Error('rate limit exceeded'))).toBeNull()+ })++ it('returns names and library identifiers for an unclassified error', () => {+ const cause = Object.assign(new Error('private cause message'), {+ name: 'ProviderFailure',+ code: 'CAUSE_CODE'+ })+ const error = Object.assign(new Error('private top-level message'), {+ name: 'SDKFailure',+ code: 'SDK_CODE',+ errno: 73,+ cause+ })++ expect(buildStreamErrorShape(error)).toEqual({+ name: 'SDKFailure',+ code: 'SDK_CODE',+ errno: 73,+ cause: {+ name: 'ProviderFailure',+ code: 'CAUSE_CODE'+ }+ })+ })++ it('never includes message text and caps string identifiers', () => {+ const sentinel = 'PRIVATE_USER_CONTENT'+ const error = Object.assign(new Error(sentinel), {+ name: `Failure${'x'.repeat(200)}`,+ code: `CODE${'y'.repeat(200)}`,+ errno: `ERRNO${'z'.repeat(200)}`+ })++ const shape = buildStreamErrorShape(error)+ const output = JSON.stringify(shape)++ expect(output).not.toContain(sentinel)+ expect(shape?.name).toHaveLength(128)+ expect(shape?.code).toHaveLength(128)+ expect(shape?.errno).toHaveLength(128)+ })+})++describe('buildStreamErrorSpanUpdate', () => {+ it('returns null for an aborted turn', () => {+ const error = new Error('request stopped')+ error.name = 'AbortError'++ expect(buildStreamErrorSpanUpdate(error, true)).toBeNull()+ })++ it('keeps an abort-named failure on the error surface when the request was not cancelled', () => {+ const error = new Error('upstream timed out')+ error.name = 'AbortError'++ expect(buildStreamErrorSpanUpdate(error, false)?.level).toBe('ERROR')+ })++ it('carries no metadata for a classified failure', () => {+ const update = buildStreamErrorSpanUpdate(+ new Error('rate limit exceeded'),+ false+ )++ expect(update?.level).toBe('ERROR')+ expect(update?.statusMessage).toContain('provider_rate_limit')+ expect(update).not.toHaveProperty('metadata')+ })+})diff --git a/lib/streaming/helpers/stream-error-diagnostics.ts b/lib/streaming/helpers/stream-error-diagnostics.tsnew file mode 100644index 00000000..7e72aa10--- /dev/null+++ b/lib/streaming/helpers/stream-error-diagnostics.ts@@ -0,0 +1,117 @@+import { toPublicErrorPayload } from '@/lib/errors/public-error'++import { describeStreamError } from './describe-stream-error'+import { buildAPICallErrorDiagnostics } from './log-api-call-error'++const MAX_IDENTIFIER_LENGTH = 128++function isObject(value: unknown): value is object {+ return typeof value === 'object' && value !== null+}++// Matched on the error name only: the tool layer aborts its own controller on a+// timeout, and that failure must stay on the error surface.+function isNamedAbortError(error: unknown): boolean {+ if (!isObject(error)) return false++ const isError =+ error instanceof Error ||+ (typeof DOMException !== 'undefined' && error instanceof DOMException)++ return (+ isError && (error.name === 'AbortError' || error.name === 'ResponseAborted')+ )+}++export function isStreamAbortError(error: unknown): boolean {+ if (isNamedAbortError(error)) return true+ if (!isObject(error)) return false++ return isNamedAbortError(Reflect.get(error, 'cause'))+}++function capIdentifier(value: unknown): string | number | undefined {+ if (typeof value === 'string') return value.slice(0, MAX_IDENTIFIER_LENGTH)++ return typeof value === 'number' ? value : undefined+}++function getName(value: unknown): string | undefined {+ if (!isObject(value)) return typeof value++ const name = Reflect.get(value, 'name')+ if (typeof name === 'string' && name) {+ return name.slice(0, MAX_IDENTIFIER_LENGTH)+ }++ const constructorName = value.constructor?.name+ return typeof constructorName === 'string' && constructorName+ ? constructorName.slice(0, MAX_IDENTIFIER_LENGTH)+ : undefined+}++function getShape(value: unknown) {+ const name = getName(value)+ const code = isObject(value)+ ? capIdentifier(Reflect.get(value, 'code'))+ : undefined+ const errno = isObject(value)+ ? capIdentifier(Reflect.get(value, 'errno'))+ : undefined++ return {+ ...(name && { name }),+ ...(code !== undefined && { code }),+ ...(errno !== undefined && { errno })+ }+}++/**+ * Identifiers and shapes only, following the discipline of+ * `buildAPICallErrorDiagnostics`: no message, no stack, nothing derived from+ * user input. Only an unclassified failure needs this, since every other code+ * is already its own signature in the status message.+ */+export function buildStreamErrorShape(+ error: unknown+): Record<string, unknown> | null {+ try {+ if (toPublicErrorPayload(error).code !== 'unknown') return null++ const cause = isObject(error) ? Reflect.get(error, 'cause') : undefined+ const causeShape = isObject(cause) ? getShape(cause) : undefined++ return {+ ...getShape(error),+ ...(causeShape &&+ Object.keys(causeShape).length > 0 && { cause: causeShape })+ }+ } catch {+ return null+ }+}++export function buildStreamErrorSpanUpdate(+ error: unknown,+ requestWasCancelled: boolean+) {+ // A cancelled turn is the user walking away, not a failure. The name alone+ // does not prove that: an internal timeout aborts its own signal and raises+ // the same one, so the request must have been cancelled too before the+ // failure is dropped from the error surface.+ if (requestWasCancelled && isStreamAbortError(error)) return null++ const apiCallDiagnostics = buildAPICallErrorDiagnostics(error)+ const streamErrorShape = buildStreamErrorShape(error)++ return {+ level: 'ERROR' as const,+ statusMessage: describeStreamError(error),+ ...((apiCallDiagnostics || streamErrorShape) && {+ metadata: {+ ...(apiCallDiagnostics && { apiCallDiagnostics }),+ ...(streamErrorShape && { streamErrorShape })+ }+ })+ }+}