Skip to content

An AI-powered search engine with a generative UI

fix(tracing): record whether a stream failure happened before the first generation (#965)

Yoshiki Miura116bae762107
diff --git a/lib/streaming/__tests__/create-chat-stream-response.test.ts b/lib/streaming/__tests__/create-chat-stream-response.test.ts
index 6cbcc5a5..2bf56f6a 100644
--- a/lib/streaming/__tests__/create-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-chat-stream-response.test.ts
@@ -69,6 +69,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({
import { researcher } from '@/lib/agents/researcher'
import { createChatStreamResponse } from '@/lib/streaming/create-chat-stream-response'
import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'
+import { prepareMessages } from '@/lib/streaming/helpers/prepare-messages'
type StreamOptions = {
onError: (event: { error: unknown }) => void
@@ -177,8 +178,28 @@ describe('createChatStreamResponse', () => {
expect(mocks.span.update).toHaveBeenCalledWith({
level: 'ERROR',
statusMessage: describeStreamError(streamError),
- metadata: { streamErrorShape: { name: 'AbortError' } }
+ metadata: {
+ streamErrorPhase: 'generation',
+ streamErrorShape: { name: 'AbortError' }
+ }
+ })
+ })
+
+ it('marks a failure raised before the stream as the preparation phase', async () => {
+ const prepareError = new TypeError('private failure detail')
+ vi.mocked(prepareMessages).mockRejectedValueOnce(prepareError)
+
+ await createChatStreamResponse(createConfig())
+
+ expect(mocks.span.update).toHaveBeenCalledWith({
+ level: 'ERROR',
+ statusMessage: describeStreamError(prepareError),
+ metadata: {
+ streamErrorPhase: 'preparation',
+ streamErrorShape: { name: 'TypeError' }
+ }
})
+ expect(mocks.stream).not.toHaveBeenCalled()
})
it('attaches shape metadata for an unclassified stream error', async () => {
@@ -198,6 +219,7 @@ describe('createChatStreamResponse', () => {
level: 'ERROR',
statusMessage: describeStreamError(streamError),
metadata: {
+ streamErrorPhase: 'generation',
streamErrorShape: {
name: 'StreamFailure',
errno: 91
diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
index 092729c0..52ce09f0 100644
--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
@@ -185,6 +185,7 @@ describe('createEphemeralChatStreamResponse', () => {
level: 'ERROR',
statusMessage: describeStreamError(streamError),
metadata: {
+ streamErrorPhase: 'generation',
streamErrorShape: {
name: 'StreamFailure',
code: 'STREAM_CODE'
@@ -258,6 +259,7 @@ describe('createEphemeralChatStreamResponse', () => {
level: 'ERROR',
statusMessage: describeStreamError(streamError),
metadata: {
+ streamErrorPhase: 'generation',
streamErrorShape: { name: 'Error' }
}
})
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index b28a559f..f3f5ad0e 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -38,7 +38,10 @@ import {
import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
import { persistStreamResults } from './helpers/persist-stream-results'
import { prepareMessages } from './helpers/prepare-messages'
-import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'
+import {
+ buildStreamErrorSpanUpdate,
+ type StreamErrorPhase
+} from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import type { StreamContext } from './helpers/types'
import { BaseStreamConfig } from './types'
@@ -97,6 +100,8 @@ export async function createChatStreamResponse(
let hasStreamError = false
let hasEmptyResponse = false
let streamError: unknown
+ // The agent stream call is preparation until its first generation starts.
+ let streamErrorPhase: StreamErrorPhase = 'preparation'
// Sampled where the failure happens: the client can disconnect before the
// span is closed, and by then the signal no longer says whether the user
// cancelled this turn or the failure was the model's own.
@@ -107,7 +112,8 @@ export async function createChatStreamResponse(
if (hasStreamError) {
const update = buildStreamErrorSpanUpdate(
streamError,
- streamErrorWasCancelled
+ streamErrorWasCancelled,
+ streamErrorPhase
)
if (update) rootSpan.update(update)
} else if (hasEmptyResponse) {
@@ -209,6 +215,7 @@ export async function createChatStreamResponse(
hasStreamError = true
streamError = error
streamErrorWasCancelled = abortSignal?.aborted ?? false
+ streamErrorPhase = 'generation'
},
experimental_transform: smoothStream({ chunking: 'word' }),
...(isUsageLogging() && {
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 6ba9c2e0..013151a2 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -31,7 +31,10 @@ import {
isEmptyResponse
} from './helpers/is-empty-response'
import { logAPICallErrorDiagnostics } from './helpers/log-api-call-error'
-import { buildStreamErrorSpanUpdate } from './helpers/stream-error-diagnostics'
+import {
+ buildStreamErrorSpanUpdate,
+ type StreamErrorPhase
+} from './helpers/stream-error-diagnostics'
import { stripSpecFromMessages } from './helpers/strip-spec-from-messages'
import { BaseStreamConfig } from './types'
@@ -64,6 +67,8 @@ export async function createEphemeralChatStreamResponse(
let hasStreamError = false
let hasEmptyResponse = false
let streamError: unknown
+ // The agent stream call is preparation until its first generation starts.
+ let streamErrorPhase: StreamErrorPhase = 'preparation'
// Sampled where the failure happens: the client can disconnect before the
// span is closed, and by then the signal no longer says whether the user
// cancelled this turn or the failure was the model's own.
@@ -74,7 +79,8 @@ export async function createEphemeralChatStreamResponse(
if (hasStreamError) {
const update = buildStreamErrorSpanUpdate(
streamError,
- streamErrorWasCancelled
+ streamErrorWasCancelled,
+ streamErrorPhase
)
if (update) rootSpan.update(update)
} else if (hasEmptyResponse) {
@@ -121,6 +127,7 @@ export async function createEphemeralChatStreamResponse(
hasStreamError = true
streamError = error
streamErrorWasCancelled = abortSignal?.aborted ?? false
+ streamErrorPhase = 'generation'
},
experimental_transform: smoothStream({ chunking: 'word' }),
...(isUsageLogging() && {
diff --git a/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts
index eff90160..f2e6e842 100644
--- a/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts
+++ b/lib/streaming/helpers/__tests__/stream-error-diagnostics.test.ts
@@ -84,28 +84,47 @@ describe('buildStreamErrorShape', () => {
})
describe('buildStreamErrorSpanUpdate', () => {
- it('returns null for an aborted turn', () => {
- const error = new Error('request stopped')
- error.name = 'AbortError'
-
- expect(buildStreamErrorSpanUpdate(error, true)).toBeNull()
- })
+ it.each(['preparation', 'generation'] as const)(
+ 'returns null for an aborted turn in the %s phase',
+ phase => {
+ const error = new Error('request stopped')
+ error.name = 'AbortError'
+
+ expect(buildStreamErrorSpanUpdate(error, true, phase)).toBeNull()
+ }
+ )
+
+ it.each(['preparation', 'generation'] as const)(
+ 'records the %s phase',
+ phase => {
+ const update = buildStreamErrorSpanUpdate(
+ new Error('unclassified failure'),
+ false,
+ phase
+ )
+
+ expect(update?.metadata.streamErrorPhase).toBe(phase)
+ }
+ )
it('keeps an abort-named failure on the error surface when the request was not cancelled', () => {
const error = new Error('upstream timed out')
error.name = 'AbortError'
- expect(buildStreamErrorSpanUpdate(error, false)?.level).toBe('ERROR')
+ expect(buildStreamErrorSpanUpdate(error, false, 'generation')?.level).toBe(
+ 'ERROR'
+ )
})
- it('carries no metadata for a classified failure', () => {
+ it('records the phase when other diagnostics are absent', () => {
const update = buildStreamErrorSpanUpdate(
new Error('rate limit exceeded'),
- false
+ false,
+ 'preparation'
)
expect(update?.level).toBe('ERROR')
expect(update?.statusMessage).toContain('provider_rate_limit')
- expect(update).not.toHaveProperty('metadata')
+ expect(update?.metadata).toEqual({ streamErrorPhase: 'preparation' })
})
})
diff --git a/lib/streaming/helpers/stream-error-diagnostics.ts b/lib/streaming/helpers/stream-error-diagnostics.ts
index 7e72aa10..bb22d87f 100644
--- a/lib/streaming/helpers/stream-error-diagnostics.ts
+++ b/lib/streaming/helpers/stream-error-diagnostics.ts
@@ -5,6 +5,8 @@ import { buildAPICallErrorDiagnostics } from './log-api-call-error'
const MAX_IDENTIFIER_LENGTH = 128
+export type StreamErrorPhase = 'preparation' | 'generation'
+
function isObject(value: unknown): value is object {
return typeof value === 'object' && value !== null
}
@@ -93,7 +95,8 @@ export function buildStreamErrorShape(
export function buildStreamErrorSpanUpdate(
error: unknown,
- requestWasCancelled: boolean
+ requestWasCancelled: boolean,
+ streamErrorPhase: StreamErrorPhase
) {
// A cancelled turn is the user walking away, not a failure. The name alone
// does not prove that: an internal timeout aborts its own signal and raises
@@ -107,11 +110,10 @@ export function buildStreamErrorSpanUpdate(
return {
level: 'ERROR' as const,
statusMessage: describeStreamError(error),
- ...((apiCallDiagnostics || streamErrorShape) && {
- metadata: {
- ...(apiCallDiagnostics && { apiCallDiagnostics }),
- ...(streamErrorShape && { streamErrorShape })
- }
- })
+ metadata: {
+ streamErrorPhase,
+ ...(apiCallDiagnostics && { apiCallDiagnostics }),
+ ...(streamErrorShape && { streamErrorShape })
+ }
}
}