Skip to content

An AI-powered search engine with a generative UI

fix(tracing): distinguish unclassified stream failures from one another and from user aborts (#956)

Yoshiki Miura6c051a99bd93
diff --git a/lib/streaming/__tests__/create-chat-stream-response.test.ts b/lib/streaming/__tests__/create-chat-stream-response.test.ts
new file mode 100644
index 00000000..4220d07a
--- /dev/null
+++ b/lib/streaming/__tests__/create-chat-stream-response.test.ts
@@ -0,0 +1,199 @@
+import { beforeEach, describe, expect, it, vi } from 'vitest'
+
+const mocks = vi.hoisted(() => ({
+ stream: vi.fn(),
+ span: {
+ traceId: 'trace-id',
+ update: vi.fn(),
+ end: vi.fn()
+ },
+ forceFlush: vi.fn(),
+ finishPromise: Promise.resolve()
+}))
+
+vi.mock('ai', () => ({
+ consumeStream: vi.fn(),
+ convertToModelMessages: vi.fn(async () => []),
+ smoothStream: vi.fn()
+}))
+
+vi.mock('@langfuse/tracing', () => ({
+ propagateAttributes: vi.fn((_attributes: unknown, callback: () => unknown) =>
+ callback()
+ ),
+ startActiveObservation: vi.fn(
+ (_name: string, callback: (span: typeof mocks.span) => unknown) =>
+ callback(mocks.span)
+ )
+}))
+
+vi.mock('@/instrumentation', () => ({
+ langfuseSpanProcessor: {
+ forceFlush: mocks.forceFlush
+ }
+}))
+
+vi.mock('@/lib/actions/chat', () => ({
+ loadChat: vi.fn()
+}))
+
+vi.mock('@/lib/agents/researcher', () => ({
+ researcher: vi.fn(() => ({ stream: mocks.stream }))
+}))
+
+vi.mock('@/lib/agents/title-generator', () => ({
+ generateChatTitle: vi.fn(async () => 'Title')
+}))
+
+vi.mock('@/lib/streaming/helpers/attachment-sizes', () => ({
+ resolveAttachmentSizes: vi.fn(async (messages: unknown) => messages)
+}))
+
+vi.mock('@/lib/streaming/helpers/persist-stream-results', () => ({
+ persistStreamResults: vi.fn(async () => undefined)
+}))
+
+vi.mock('@/lib/streaming/helpers/prepare-messages', () => ({
+ prepareMessages: vi.fn(async () => [])
+}))
+
+vi.mock('@/lib/utils/telemetry', () => ({
+ isTracingEnabled: vi.fn(() => true)
+}))
+
+vi.mock('@/lib/utils/usage-logging', () => ({
+ isUsageLogging: vi.fn(() => false),
+ logUsage: vi.fn()
+}))
+
+import { createChatStreamResponse } from '@/lib/streaming/create-chat-stream-response'
+import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'
+
+type StreamOptions = {
+ onError: (event: { error: unknown }) => void
+}
+
+type UIMessageStreamResponseOptions = {
+ onFinish: (event: {
+ responseMessage: {
+ id: string
+ role: 'assistant'
+ parts: Array<{ type: string; text?: string }>
+ }
+ isAborted: boolean
+ }) => Promise<void>
+}
+
+function createFakeResult(isAborted = false) {
+ return {
+ consumeStream: vi.fn(),
+ toUIMessageStreamResponse: vi.fn(
+ (options: UIMessageStreamResponseOptions) => {
+ mocks.finishPromise = options.onFinish({
+ responseMessage: {
+ id: 'response-id',
+ role: 'assistant',
+ parts: [{ type: 'text', text: 'Answer' }]
+ },
+ isAborted
+ })
+ return new Response()
+ }
+ )
+ }
+}
+
+function createAbortedSignal(): AbortSignal {
+ const controller = new AbortController()
+ controller.abort()
+
+ return controller.signal
+}
+
+function createConfig(abortSignal: AbortSignal = new AbortController().signal) {
+ return {
+ message: {
+ id: 'message-id',
+ role: 'user' as const,
+ parts: [{ type: 'text' as const, text: 'hello' }]
+ },
+ model: { providerId: 'openai', id: 'gpt-4o-mini' } as any,
+ chatId: 'chat-id',
+ userId: 'user-id',
+ abortSignal,
+ isNewChat: true,
+ searchMode: 'quick' as const
+ }
+}
+
+describe('createChatStreamResponse', () => {
+ beforeEach(() => {
+ vi.clearAllMocks()
+ mocks.finishPromise = Promise.resolve()
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ })
+
+ it('does not mark the span as failed for an aborted stream error', async () => {
+ const streamError = new Error('request stopped')
+ streamError.name = 'ResponseAborted'
+ mocks.stream.mockImplementation(async (options: StreamOptions) => {
+ options.onError({ error: streamError })
+ return createFakeResult(true)
+ })
+
+ await createChatStreamResponse(createConfig(createAbortedSignal()))
+ await mocks.finishPromise
+
+ expect(mocks.span.update).not.toHaveBeenCalled()
+ expect(mocks.span.end).toHaveBeenCalledOnce()
+ expect(mocks.forceFlush).toHaveBeenCalledOnce()
+ })
+
+ it('marks the span as failed when the client disconnects after the failure', async () => {
+ const streamError = new Error('upstream stopped')
+ streamError.name = 'AbortError'
+ const controller = new AbortController()
+ mocks.stream.mockImplementation(async (options: StreamOptions) => {
+ options.onError({ error: streamError })
+ controller.abort()
+ return createFakeResult()
+ })
+
+ await createChatStreamResponse(createConfig(controller.signal))
+ await mocks.finishPromise
+
+ expect(mocks.span.update).toHaveBeenCalledWith({
+ level: 'ERROR',
+ statusMessage: describeStreamError(streamError),
+ metadata: { streamErrorShape: { name: 'AbortError' } }
+ })
+ })
+
+ it('attaches shape metadata for an unclassified stream error', async () => {
+ const streamError = Object.assign(new Error('private failure detail'), {
+ name: 'StreamFailure',
+ errno: 91
+ })
+ mocks.stream.mockImplementation(async (options: StreamOptions) => {
+ options.onError({ error: streamError })
+ return createFakeResult()
+ })
+
+ await createChatStreamResponse(createConfig())
+ await mocks.finishPromise
+
+ expect(mocks.span.update).toHaveBeenCalledWith({
+ level: 'ERROR',
+ statusMessage: describeStreamError(streamError),
+ metadata: {
+ streamErrorShape: {
+ name: 'StreamFailure',
+ errno: 91
+ }
+ }
+ })
+ expect(JSON.stringify(mocks.span.update.mock.calls)).not.toContain(
+ 'private failure detail'
+ )
+ })
+})
diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
index 3788ad14..d79cf289 100644
--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
@@ -98,7 +98,14 @@ function createFakeResult({
}
}
-function createConfig() {
+function createAbortedSignal(): AbortSignal {
+ const controller = new AbortController()
+ controller.abort()
+
+ return controller.signal
+}
+
+function createConfig(abortSignal: AbortSignal = new AbortController().signal) {
return {
messages: [
{
@@ -108,7 +115,7 @@ function createConfig() {
}
],
model: { providerId: 'openai', id: 'gpt-4o-mini' } as any,
- abortSignal: new AbortController().signal,
+ abortSignal,
searchMode: 'quick' as const
}
}
@@ -148,8 +155,11 @@ describe('createEphemeralChatStreamResponse', () => {
expect(mocks.forceFlush).toHaveBeenCalledOnce()
})
- it('marks the span as failed for a stream-level error', async () => {
- const streamError = new Error('stream failed')
+ it('attaches shape metadata for an unclassified stream error', async () => {
+ const streamError = Object.assign(new Error('stream failed'), {
+ name: 'StreamFailure',
+ code: 'STREAM_CODE'
+ })
mocks.stream.mockImplementation(async (options: StreamOptions) => {
options.onError({ error: streamError })
return createFakeResult()
@@ -160,12 +170,34 @@ describe('createEphemeralChatStreamResponse', () => {
expect(mocks.span.update).toHaveBeenCalledWith({
level: 'ERROR',
- statusMessage: describeStreamError(streamError)
+ statusMessage: describeStreamError(streamError),
+ metadata: {
+ streamErrorShape: {
+ name: 'StreamFailure',
+ code: 'STREAM_CODE'
+ }
+ }
})
expect(mocks.span.end).toHaveBeenCalledOnce()
expect(mocks.forceFlush).toHaveBeenCalledOnce()
})
+ it('does not mark the span as failed for an aborted stream error', async () => {
+ const streamError = new Error('request stopped')
+ streamError.name = 'AbortError'
+ mocks.stream.mockImplementation(async (options: StreamOptions) => {
+ options.onError({ error: streamError })
+ return createFakeResult({ isAborted: true })
+ })
+
+ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))
+ await mocks.finishPromise
+
+ expect(mocks.span.update).not.toHaveBeenCalled()
+ expect(mocks.span.end).toHaveBeenCalledOnce()
+ expect(mocks.forceFlush).toHaveBeenCalledOnce()
+ })
+
it('marks the span as failed for an empty response', async () => {
mocks.stream.mockResolvedValue(createFakeResult({ parts: [] }))
@@ -181,7 +213,7 @@ describe('createEphemeralChatStreamResponse', () => {
it('does not update the span for a response with text', async () => {
mocks.stream.mockResolvedValue(createFakeResult())
- await createEphemeralChatStreamResponse(createConfig())
+ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))
await mocks.finishPromise
expect(mocks.span.update).not.toHaveBeenCalled()
@@ -192,7 +224,7 @@ describe('createEphemeralChatStreamResponse', () => {
createFakeResult({ parts: [], isAborted: true })
)
- await createEphemeralChatStreamResponse(createConfig())
+ await createEphemeralChatStreamResponse(createConfig(createAbortedSignal()))
await mocks.finishPromise
expect(mocks.span.update).not.toHaveBeenCalled()
@@ -211,7 +243,10 @@ describe('createEphemeralChatStreamResponse', () => {
expect(mocks.span.update).toHaveBeenCalledOnce()
expect(mocks.span.update).toHaveBeenCalledWith({
level: 'ERROR',
- statusMessage: describeStreamError(streamError)
+ statusMessage: describeStreamError(streamError),
+ metadata: {
+ streamErrorShape: { name: 'Error' }
+ }
})
})
})
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 5c3005bb..cd046eb0 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -31,17 +31,14 @@ import { compactHistoricalMessages } from './helpers/compact-historical-messages
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
import { dedupeAttachments } from './helpers/dedupe-attachments'
-import { describeStreamError } from './helpers/describe-stream-error'
import {
EMPTY_RESPONSE_STATUS_MESSAGE,
isEmptyResponse
} from './helpers/is-empty-response'
-import {
- buildAPICallErrorDiagnostics,
- logAPICallErrorDiagnostics
-} from './helpers/log-api-call-error'
+import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
import { persistStreamResults } from './helpers/persist-stream-results'
import { prepareMessages } from './helpers/prepare-messages'
+import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import type { StreamContext } from './helpers/types'
import { BaseStreamConfig } from './types'
@@ -100,16 +97,19 @@ export async function createChatStreamResponse(
let hasStreamError = false
let hasEmptyResponse = false
let streamError: unknown
+ // Sampled where the failure happens: the client can disconnect before the
+ // span is closed, and by then the signal no longer says whether the user
+ // cancelled this turn or the failure was the model's own.
+ let streamErrorWasCancelled = false
const endTracing = async () => {
if (rootSpan) {
if (hasStreamError) {
- const apiCallDiagnostics = buildAPICallErrorDiagnostics(streamError)
- rootSpan.update({
- level: 'ERROR',
- statusMessage: describeStreamError(streamError),
- ...(apiCallDiagnostics && { metadata: { apiCallDiagnostics } })
- })
+ const update = buildStreamErrorSpanUpdate(
+ streamError,
+ streamErrorWasCancelled
+ )
+ if (update) rootSpan.update(update)
} else if (hasEmptyResponse) {
rootSpan.update({
level: 'ERROR',
@@ -207,6 +207,7 @@ export async function createChatStreamResponse(
onError: ({ error }) => {
hasStreamError = true
streamError = error
+ streamErrorWasCancelled = abortSignal?.aborted ?? false
},
experimental_transform: smoothStream({ chunking: 'word' }),
...(isUsageLogging() && {
@@ -281,6 +282,7 @@ export async function createChatStreamResponse(
} catch (error) {
hasStreamError = true
streamError = error
+ streamErrorWasCancelled = abortSignal?.aborted ?? false
await endTracing()
logAPICallErrorDiagnostics(error)
console.error('Stream execution error:', error)
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 9d90e527..6d5152a5 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -26,15 +26,12 @@ import { compactHistoricalMessages } from './helpers/compact-historical-messages
import { convertDataPart } from './helpers/convert-data-part'
import { assignDataPartNonces } from './helpers/data-part-nonce'
import { dedupeAttachments } from './helpers/dedupe-attachments'
-import { describeStreamError } from './helpers/describe-stream-error'
import {
EMPTY_RESPONSE_STATUS_MESSAGE,
isEmptyResponse
} from './helpers/is-empty-response'
-import {
- buildAPICallErrorDiagnostics,
- logAPICallErrorDiagnostics
-} from './helpers/log-api-call-error'
+import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
+import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import { BaseStreamConfig } from './types'
@@ -67,16 +64,19 @@ export async function createEphemeralChatStreamResponse(
let hasStreamError = false
let hasEmptyResponse = false
let streamError: unknown
+ // Sampled where the failure happens: the client can disconnect before the
+ // span is closed, and by then the signal no longer says whether the user
+ // cancelled this turn or the failure was the model's own.
+ let streamErrorWasCancelled = false
const endTracing = async () => {
if (rootSpan) {
if (hasStreamError) {
- const apiCallDiagnostics = buildAPICallErrorDiagnostics(streamError)
- rootSpan.update({
- level: 'ERROR',
- statusMessage: describeStreamError(streamError),
- ...(apiCallDiagnostics && { metadata: { apiCallDiagnostics } })
- })
+ const update = buildStreamErrorSpanUpdate(
+ streamError,
+ streamErrorWasCancelled
+ )
+ if (update) rootSpan.update(update)
} else if (hasEmptyResponse) {
rootSpan.update({
level: 'ERROR',
@@ -119,6 +119,7 @@ export async function createEphemeralChatStreamResponse(
onError: ({ error }) => {
hasStreamError = true
streamError = error
+ streamErrorWasCancelled = abortSignal?.aborted ?? false
},
experimental_transform: smoothStream({ chunking: 'word' }),
...(isUsageLogging() && {
@@ -172,6 +173,7 @@ export async function createEphemeralChatStreamResponse(
} catch (error) {
hasStreamError = true
streamError = error
+ streamErrorWasCancelled = abortSignal?.aborted ?? false
await endTracing()
logAPICallErrorDiagnostics(error)
console.error('Ephemeral stream execution error:', error)
diff --git a/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts
new file mode 100644
index 00000000..eff90160
--- /dev/null
+++ b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts
@@ -0,0 +1,111 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ buildStreamErrorShape,
+ buildStreamErrorSpanUpdate,
+ isStreamAbortError
+} from '../stream-error-diagnostics'
+
+describe('isStreamAbortError', () => {
+ it.each(['AbortError', 'ResponseAborted'])('recognizes %s errors', name => {
+ const error = new Error('request stopped')
+ error.name = name
+
+ expect(isStreamAbortError(error)).toBe(true)
+ })
+
+ it('recognizes an abort DOMException', () => {
+ expect(
+ isStreamAbortError(new DOMException('request stopped', 'AbortError'))
+ ).toBe(true)
+ })
+
+ it('recognizes an abort error in the cause', () => {
+ const cause = new Error('request stopped')
+ cause.name = 'ResponseAborted'
+
+ expect(isStreamAbortError(new Error('wrapper', { cause }))).toBe(true)
+ })
+
+ it.each([
+ new Error('AbortError'),
+ Object.assign(new Error('tool timed out'), { name: 'TimeoutError' }),
+ { name: 'AbortError' },
+ new Error('ordinary failure')
+ ])('does not recognize non-abort errors', error => {
+ expect(isStreamAbortError(error)).toBe(false)
+ })
+})
+
+describe('buildStreamErrorShape', () => {
+ it('returns null for a classified error', () => {
+ expect(buildStreamErrorShape(new Error('rate limit exceeded'))).toBeNull()
+ })
+
+ it('returns names and library identifiers for an unclassified error', () => {
+ const cause = Object.assign(new Error('private cause message'), {
+ name: 'ProviderFailure',
+ code: 'CAUSE_CODE'
+ })
+ const error = Object.assign(new Error('private top-level message'), {
+ name: 'SDKFailure',
+ code: 'SDK_CODE',
+ errno: 73,
+ cause
+ })
+
+ expect(buildStreamErrorShape(error)).toEqual({
+ name: 'SDKFailure',
+ code: 'SDK_CODE',
+ errno: 73,
+ cause: {
+ name: 'ProviderFailure',
+ code: 'CAUSE_CODE'
+ }
+ })
+ })
+
+ it('never includes message text and caps string identifiers', () => {
+ const sentinel = 'PRIVATE_USER_CONTENT'
+ const error = Object.assign(new Error(sentinel), {
+ name: `Failure${'x'.repeat(200)}`,
+ code: `CODE${'y'.repeat(200)}`,
+ errno: `ERRNO${'z'.repeat(200)}`
+ })
+
+ const shape = buildStreamErrorShape(error)
+ const output = JSON.stringify(shape)
+
+ expect(output).not.toContain(sentinel)
+ expect(shape?.name).toHaveLength(128)
+ expect(shape?.code).toHaveLength(128)
+ expect(shape?.errno).toHaveLength(128)
+ })
+})
+
+describe('buildStreamErrorSpanUpdate', () => {
+ it('returns null for an aborted turn', () => {
+ const error = new Error('request stopped')
+ error.name = 'AbortError'
+
+ expect(buildStreamErrorSpanUpdate(error, true)).toBeNull()
+ })
+
+ it('keeps an abort-named failure on the error surface when the request was not cancelled', () => {
+ const error = new Error('upstream timed out')
+ error.name = 'AbortError'
+
+ expect(buildStreamErrorSpanUpdate(error, false)?.level).toBe('ERROR')
+ })
+
+ it('carries no metadata for a classified failure', () => {
+ const update = buildStreamErrorSpanUpdate(
+ new Error('rate limit exceeded'),
+ false
+ )
+
+ expect(update?.level).toBe('ERROR')
+ expect(update?.statusMessage).toContain('provider_rate_limit')
+ expect(update).not.toHaveProperty('metadata')
+ })
+})
diff --git a/lib/streaming/helpers/stream-error-diagnostics.ts b/lib/streaming/helpers/stream-error-diagnostics.ts
new file mode 100644
index 00000000..7e72aa10
--- /dev/null
+++ b/lib/streaming/helpers/stream-error-diagnostics.ts
@@ -0,0 +1,117 @@
+import { toPublicErrorPayload } from '@/lib/errors/public-error'
+
+import { describeStreamError } from './describe-stream-error'
+import { buildAPICallErrorDiagnostics } from './log-api-call-error'
+
+const MAX_IDENTIFIER_LENGTH = 128
+
+function isObject(value: unknown): value is object {
+ return typeof value === 'object' && value !== null
+}
+
+// Matched on the error name only: the tool layer aborts its own controller on a
+// timeout, and that failure must stay on the error surface.
+function isNamedAbortError(error: unknown): boolean {
+ if (!isObject(error)) return false
+
+ const isError =
+ error instanceof Error ||
+ (typeof DOMException !== 'undefined' && error instanceof DOMException)
+
+ return (
+ isError && (error.name === 'AbortError' || error.name === 'ResponseAborted')
+ )
+}
+
+export function isStreamAbortError(error: unknown): boolean {
+ if (isNamedAbortError(error)) return true
+ if (!isObject(error)) return false
+
+ return isNamedAbortError(Reflect.get(error, 'cause'))
+}
+
+function capIdentifier(value: unknown): string | number | undefined {
+ if (typeof value === 'string') return value.slice(0, MAX_IDENTIFIER_LENGTH)
+
+ return typeof value === 'number' ? value : undefined
+}
+
+function getName(value: unknown): string | undefined {
+ if (!isObject(value)) return typeof value
+
+ const name = Reflect.get(value, 'name')
+ if (typeof name === 'string' && name) {
+ return name.slice(0, MAX_IDENTIFIER_LENGTH)
+ }
+
+ const constructorName = value.constructor?.name
+ return typeof constructorName === 'string' && constructorName
+ ? constructorName.slice(0, MAX_IDENTIFIER_LENGTH)
+ : undefined
+}
+
+function getShape(value: unknown) {
+ const name = getName(value)
+ const code = isObject(value)
+ ? capIdentifier(Reflect.get(value, 'code'))
+ : undefined
+ const errno = isObject(value)
+ ? capIdentifier(Reflect.get(value, 'errno'))
+ : undefined
+
+ return {
+ ...(name && { name }),
+ ...(code !== undefined && { code }),
+ ...(errno !== undefined && { errno })
+ }
+}
+
+/**
+ * Identifiers and shapes only, following the discipline of
+ * `buildAPICallErrorDiagnostics`: no message, no stack, nothing derived from
+ * user input. Only an unclassified failure needs this, since every other code
+ * is already its own signature in the status message.
+ */
+export function buildStreamErrorShape(
+ error: unknown
+): Record<string, unknown> | null {
+ try {
+ if (toPublicErrorPayload(error).code !== 'unknown') return null
+
+ const cause = isObject(error) ? Reflect.get(error, 'cause') : undefined
+ const causeShape = isObject(cause) ? getShape(cause) : undefined
+
+ return {
+ ...getShape(error),
+ ...(causeShape &&
+ Object.keys(causeShape).length > 0 && { cause: causeShape })
+ }
+ } catch {
+ return null
+ }
+}
+
+export function buildStreamErrorSpanUpdate(
+ error: unknown,
+ requestWasCancelled: boolean
+) {
+ // A cancelled turn is the user walking away, not a failure. The name alone
+ // does not prove that: an internal timeout aborts its own signal and raises
+ // the same one, so the request must have been cancelled too before the
+ // failure is dropped from the error surface.
+ if (requestWasCancelled && isStreamAbortError(error)) return null
+
+ const apiCallDiagnostics = buildAPICallErrorDiagnostics(error)
+ const streamErrorShape = buildStreamErrorShape(error)
+
+ return {
+ level: 'ERROR' as const,
+ statusMessage: describeStreamError(error),
+ ...((apiCallDiagnostics || streamErrorShape) && {
+ metadata: {
+ ...(apiCallDiagnostics && { apiCallDiagnostics }),
+ ...(streamErrorShape && { streamErrorShape })
+ }
+ })
+ }
+}