Skip to content

An AI-powered search engine with a generative UI

fix(errors): describe a failed tool call by its resource, not by the app (#949)

Yoshiki Miurabab0eb2a0c05
diff --git a/lib/errors/__tests__/public-error.test.ts b/lib/errors/__tests__/public-error.test.ts
index f5c3a39c..bf6c628c 100644
--- a/lib/errors/__tests__/public-error.test.ts
+++ b/lib/errors/__tests__/public-error.test.ts
@@ -6,6 +6,7 @@ import {
serializePublicError,
toPublicErrorPayload
} from '@/lib/errors/public-error'
+import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'
describe('public error mapping', () => {
it('hides provider billing errors', () => {
@@ -171,6 +172,42 @@ describe('public error mapping', () => {
expect(serialized).not.toContain('Redis timeout')
})
+ it('preserves serialized tool failure copy when parsing it again', () => {
+ const serialized = serializeToolFailure(
+ new ToolFailureError('fetch', 'HTTP 403: Forbidden')
+ )
+ const payload = toPublicErrorPayload(serialized)
+
+ expect(payload.code).toBe('tool_failed')
+ expect(payload.type).toBe('general')
+ expect(payload.error).toBe('The page refused the request.')
+ })
+
+ it('does not preserve upstream copy that only claims the tool_failed code', () => {
+ const error = new Error(
+ JSON.stringify({
+ code: 'tool_failed',
+ error: 'connection to db-primary failed for user svc_admin'
+ })
+ )
+ const payload = toPublicErrorPayload(error)
+
+ expect(payload.error).not.toContain('svc_admin')
+ expect(payload.error).not.toContain('db-primary')
+ })
+
+ it('keeps request-level 403 failures on the permission path', () => {
+ const error = Object.assign(new Error('Request failed'), {
+ statusCode: 403
+ })
+ const payload = toPublicErrorPayload(error)
+
+ expect(payload.code).toBe('forbidden')
+ expect(payload.error).toBe(
+ 'You do not have permission to access this resource.'
+ )
+ })
+
it('creates JSON error responses without raw messages', async () => {
const response = createPublicErrorResponse(
new Error(
diff --git a/lib/errors/__tests__/tool-error.test.ts b/lib/errors/__tests__/tool-error.test.ts
new file mode 100644
index 00000000..b7261ecd
--- /dev/null
+++ b/lib/errors/__tests__/tool-error.test.ts
@@ -0,0 +1,130 @@
+import { describe, expect, it } from 'vitest'
+
+import {
+ getToolFailureMessage,
+ isToolFailureError,
+ isToolFailureMessage,
+ serializeToolFailure,
+ ToolFailureError
+} from '@/lib/errors/tool-error'
+
+describe('tool error mapping', () => {
+ it.each([
+ ['HTTP 403: Forbidden', 'The page refused the request.'],
+ ['Forbidden by upstream', 'The page refused the request.'],
+ ['HTTP 404: Not Found', 'The page could not be found.'],
+ [
+ 'HTTP 429: Too Many Requests',
+ 'The page is rate limiting requests. Please try again shortly.'
+ ],
+ [
+ 'HTTP 500: Internal Server Error',
+ 'The page is temporarily unavailable. Please try again shortly.'
+ ],
+ [
+ 'Service unavailable',
+ 'The page is temporarily unavailable. Please try again shortly.'
+ ],
+ ['Request timeout after 10 seconds', 'The page took too long to respond.'],
+ ['The request was aborted', 'The page took too long to respond.'],
+ [
+ 'Unsupported content type: application/pdf',
+ 'The page is not in a readable text format.'
+ ],
+ [
+ 'No results returned from content extraction service',
+ 'The page did not return any readable content.'
+ ],
+ [
+ 'No data returned from Jina Reader API',
+ 'The page did not return any readable content.'
+ ]
+ ])('maps %s to scoped copy', (message, expected) => {
+ expect(getToolFailureMessage(new ToolFailureError('fetch', message))).toBe(
+ expected
+ )
+ })
+
+ it('uses a fetch-specific unknown fallback', () => {
+ expect(
+ getToolFailureMessage(new ToolFailureError('fetch', 'Unexpected failure'))
+ ).toBe('The page could not be read.')
+ })
+
+ it('uses a search-specific unknown fallback', () => {
+ expect(
+ getToolFailureMessage(
+ new ToolFailureError('search', 'Unexpected failure')
+ )
+ ).toBe('The search could not be completed.')
+ })
+
+ it('recognizes markers without relying only on instanceof', () => {
+ expect(
+ isToolFailureError({
+ isToolFailureError: true,
+ toolName: 'fetch',
+ originalMessage: 'failed'
+ })
+ ).toBe(true)
+ expect(isToolFailureError(new Error('failed'))).toBe(false)
+ })
+
+ it('scopes the copy to the search service', () => {
+ expect(
+ getToolFailureMessage(
+ new ToolFailureError('search', 'HTTP 403: Forbidden')
+ )
+ ).toBe('The search service refused the request.')
+ })
+
+ it('does not mark a refusal as retryable', () => {
+ const payload = JSON.parse(
+ serializeToolFailure(new ToolFailureError('fetch', 'HTTP 403: Forbidden'))
+ )
+
+ expect(payload.retryable).toBe(false)
+ })
+
+ it('serializes a tool_failed public payload', () => {
+ const payload = JSON.parse(
+ serializeToolFailure(new ToolFailureError('search', 'Unexpected failure'))
+ )
+
+ expect(payload).toMatchObject({
+ code: 'tool_failed',
+ type: 'general',
+ error: 'The search could not be completed.'
+ })
+ })
+})
+
+describe('tool failure copy', () => {
+ it('only recognizes sentences the mapper can produce', () => {
+ expect(isToolFailureMessage('The page refused the request.')).toBe(true)
+ expect(isToolFailureMessage('Something else entirely.')).toBe(false)
+ })
+
+ it.each([
+ ['fetch', 'HTTP 403: Forbidden'],
+ ['fetch', 'HTTP 404: Not Found'],
+ ['fetch', 'HTTP 429: Too Many Requests'],
+ ['fetch', 'HTTP 503: Service Temporarily Unavailable'],
+ ['fetch', 'Request timeout after 10 seconds'],
+ ['fetch', 'Unsupported content type: application/pdf'],
+ ['fetch', 'No results returned from content extraction service'],
+ ['fetch', 'Unexpected failure'],
+ ['search', 'HTTP 403: Forbidden'],
+ ['search', 'HTTP 429: Too Many Requests'],
+ ['search', 'Unexpected failure']
+ ] as const)(
+ 'keeps %s copy for %s inside the known set',
+ (toolName, message) => {
+ expect(
+ isToolFailureMessage(
+ getToolFailureMessage(new ToolFailureError(toolName, message))
+ )
+ ).toBe(true)
+ }
+ )
+})
diff --git a/lib/errors/public-error.ts b/lib/errors/public-error.ts
index 139203b2..1f2ed3a9 100644
--- a/lib/errors/public-error.ts
+++ b/lib/errors/public-error.ts
@@ -1,3 +1,5 @@
+import { isToolFailureMessage } from './tool-error'
+
export type PublicErrorType = 'rate-limit' | 'auth' | 'forbidden' | 'general'
export type PublicErrorCode =
@@ -12,6 +14,7 @@ export type PublicErrorCode =
| 'provider_rate_limit'
| 'provider_unavailable'
| 'rate_limit'
+ | 'tool_failed'
| 'unknown'
export type PublicErrorPayload = {
@@ -54,6 +57,7 @@ const PUBLIC_ERROR_CODES: ReadonlySet<string> = new Set([
'provider_rate_limit',
'provider_unavailable',
'rate_limit',
+ 'tool_failed',
'unknown'
])
@@ -371,6 +375,16 @@ function classifyError(snapshot: ErrorSnapshot, fallbackMessage?: string) {
}
}
+/**
+ * A `tool_failed` code alone is not proof of provenance: an upstream error
+ * whose own text happens to be JSON reaches this same parser, and honouring
+ * the code would hand that text straight to the client. Only copy this module
+ * knows it produced is preserved.
+ */
+function isToolFailurePayload(message: string, code: PublicErrorCode): boolean {
+ return code === 'tool_failed' && isToolFailureMessage(message)
+}
+
function isIntentionalPublicPayload(
value: Record<string, unknown>,
code: PublicErrorCode
@@ -402,7 +416,9 @@ function fromParsedPayload(
)
const code = asPublicErrorCode(value.code) ?? classification.code
const type = asPublicErrorType(value.type) ?? typeForCode(code)
- const preserveMessage = isIntentionalPublicPayload(value, code)
+ const preserveMessage =
+ isIntentionalPublicPayload(value, code) ||
+ isToolFailurePayload(message, code)
return {
error: preserveMessage ? message : classification.error,
diff --git a/lib/errors/tool-error.ts b/lib/errors/tool-error.ts
new file mode 100644
index 00000000..5f2ba42f
--- /dev/null
+++ b/lib/errors/tool-error.ts
@@ -0,0 +1,161 @@
+import type { PublicErrorPayload } from './public-error'
+
+export type ToolName = 'fetch' | 'search'
+
+// What the failure happened to. The whole point of this module is that it is
+// never the user and never the AI service.
+const SUBJECT: Record<ToolName, string> = {
+ fetch: 'The page',
+ search: 'The search service'
+}
+
+function getErrorMessage(error: unknown): string {
+ return error instanceof Error ? error.message : String(error)
+}
+
+function getErrorStatus(error: unknown): number | undefined {
+ if (typeof error !== 'object' || error === null) return undefined
+
+ const record = error as Record<string, unknown>
+ const status = record.statusCode ?? record.status
+ return typeof status === 'number' && Number.isFinite(status)
+ ? status
+ : undefined
+}
+
+export class ToolFailureError extends Error {
+ readonly isToolFailureError = true
+ readonly originalMessage: string
+ readonly status?: number
+
+ constructor(
+ readonly toolName: ToolName,
+ error: unknown
+ ) {
+ const originalMessage = getErrorMessage(error)
+ super(originalMessage, { cause: error })
+ this.name = 'ToolFailureError'
+ this.originalMessage = originalMessage
+ this.status = getErrorStatus(error)
+ }
+}
+
+/**
+ * Structural rather than `instanceof`, so a failure that crossed a bundle
+ * boundary is still recognized as one.
+ */
+export function isToolFailureError(error: unknown): error is ToolFailureError {
+ if (typeof error !== 'object' || error === null) return false
+
+ const value = error as Record<string, unknown>
+ return (
+ value.isToolFailureError === true &&
+ (value.toolName === 'fetch' || value.toolName === 'search') &&
+ typeof value.originalMessage === 'string'
+ )
+}
+
+const REFUSED = (subject: string) => `${subject} refused the request.`
+const NOT_FOUND = (subject: string) => `${subject} could not be found.`
+const RATE_LIMITED = (subject: string) =>
+ `${subject} is rate limiting requests. Please try again shortly.`
+const UNAVAILABLE = (subject: string) =>
+ `${subject} is temporarily unavailable. Please try again shortly.`
+const TOO_SLOW = (subject: string) => `${subject} took too long to respond.`
+const UNREADABLE_TYPE = 'The page is not in a readable text format.'
+const NO_CONTENT = 'The page did not return any readable content.'
+const FETCH_FALLBACK = 'The page could not be read.'
+const SEARCH_FALLBACK = 'The search could not be completed.'
+
+/**
+ * Every sentence this module can produce.
+ *
+ * `public-error` preserves a `tool_failed` message instead of reclassifying it,
+ * and an error whose own text happens to be JSON is parsed by that same code
+ * path. Checking membership here means an upstream payload claiming the code
+ * can still only carry copy we wrote ourselves.
+ */
+export const TOOL_FAILURE_MESSAGES: ReadonlySet<string> = new Set([
+ ...Object.values(SUBJECT).flatMap(subject => [
+ REFUSED(subject),
+ NOT_FOUND(subject),
+ RATE_LIMITED(subject),
+ UNAVAILABLE(subject),
+ TOO_SLOW(subject)
+ ]),
+ UNREADABLE_TYPE,
+ NO_CONTENT,
+ FETCH_FALLBACK,
+ SEARCH_FALLBACK
+])
+
+export function isToolFailureMessage(message: string): boolean {
+ return TOOL_FAILURE_MESSAGES.has(message)
+}
+
+function classifyToolFailure(error: ToolFailureError): {
+ message: string
+ retryable: boolean
+} {
+ const message = error.originalMessage
+ const status = error.status
+ const subject = SUBJECT[error.toolName]
+
+ if (status === 403 || /\b403\b|\bforbidden\b/i.test(message)) {
+ return { message: REFUSED(subject), retryable: false }
+ }
+
+ if (status === 404 || /\b404\b|\bnot found\b/i.test(message)) {
+ return { message: NOT_FOUND(subject), retryable: false }
+ }
+
+ if (
+ status === 429 ||
+ /\b429\b|too many requests|rate[_\s-]?limit/i.test(message)
+ ) {
+ return { message: RATE_LIMITED(subject), retryable: true }
+ }
+
+ if (
+ (status !== undefined && status >= 500 && status <= 599) ||
+ /\b5\d{2}\b|service unavailable|bad gateway/i.test(message)
+ ) {
+ return { message: UNAVAILABLE(subject), retryable: true }
+ }
+
+ if (/timeout|timed out|abort/i.test(message)) {
+ return { message: TOO_SLOW(subject), retryable: true }
+ }
+
+ if (/unsupported content type/i.test(message)) {
+ return { message: UNREADABLE_TYPE, retryable: false }
+ }
+
+ if (
+ /no results returned from content extraction service/i.test(message) ||
+ /no data returned from jina reader api/i.test(message)
+ ) {
+ return { message: NO_CONTENT, retryable: true }
+ }
+
+ return {
+ message: error.toolName === 'fetch' ? FETCH_FALLBACK : SEARCH_FALLBACK,
+ retryable: true
+ }
+}
+
+export function getToolFailureMessage(error: ToolFailureError): string {
+ return classifyToolFailure(error).message
+}
+
+export function serializeToolFailure(error: ToolFailureError): string {
+ const { message, retryable } = classifyToolFailure(error)
+ const payload: PublicErrorPayload = {
+ error: message,
+ code: 'tool_failed',
+ type: 'general',
+ retryable
+ }
+
+ return JSON.stringify(payload)
+}
diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
index c84fe08e..3788ad14 100644
--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts
@@ -47,7 +47,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({
logUsage: vi.fn()
}))
-import { serializePublicError } from '@/lib/errors/public-error'
+import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'
import { createEphemeralChatStreamResponse } from '@/lib/streaming/create-ephemeral-chat-stream-response'
import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'
import { EMPTY_RESPONSE_STATUS_MESSAGE } from '@/lib/streaming/helpers/is-empty-response'
@@ -133,15 +133,16 @@ describe('createEphemeralChatStreamResponse', () => {
})
it('does not mark the span as failed for a tool error', async () => {
- const toolError = Object.assign(new Error('HTTP 403: Forbidden'), {
- statusCode: 403
- })
+ const toolError = new ToolFailureError(
+ 'fetch',
+ new Error('HTTP 403: Forbidden')
+ )
mocks.stream.mockResolvedValue(createFakeResult({ toolError }))
await createEphemeralChatStreamResponse(createConfig())
await mocks.finishPromise
- expect(mocks.serializedError).toBe(serializePublicError(toolError))
+ expect(mocks.serializedError).toBe(serializeToolFailure(toolError))
expect(mocks.span.update).not.toHaveBeenCalled()
expect(mocks.span.end).toHaveBeenCalledOnce()
expect(mocks.forceFlush).toHaveBeenCalledOnce()
diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.ts
index 36bdaae1..72cfc304 100644
--- a/lib/streaming/create-chat-stream-response.ts
+++ b/lib/streaming/create-chat-stream-response.ts
@@ -8,6 +8,10 @@ import {
createPublicErrorResponse,
serializePublicError
} from '@/lib/errors/public-error'
+import {
+ isToolFailureError,
+ serializeToolFailure
+} from '@/lib/errors/tool-error'
import { isTracingEnabled } from '@/lib/utils/telemetry'
import { loadChat } from '../actions/chat'
@@ -256,6 +260,11 @@ export async function createChatStreamResponse(
}
},
onError: (error: unknown) => {
+ if (isToolFailureError(error)) {
+ console.error('Tool failure:', error)
+ return serializeToolFailure(error)
+ }
+
logAPICallErrorDiagnostics(error)
console.error('Stream response error:', error)
return serializePublicError(error)
diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.ts
index 2b601d1c..9d90e527 100644
--- a/lib/streaming/create-ephemeral-chat-stream-response.ts
+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts
@@ -8,6 +8,10 @@ import {
createPublicErrorResponse,
serializePublicError
} from '@/lib/errors/public-error'
+import {
+ isToolFailureError,
+ serializeToolFailure
+} from '@/lib/errors/tool-error'
import { isTracingEnabled } from '@/lib/utils/telemetry'
import {
@@ -154,6 +158,11 @@ export async function createEphemeralChatStreamResponse(
await endTracing()
},
onError: (error: unknown) => {
+ if (isToolFailureError(error)) {
+ console.error('Ephemeral tool failure:', error)
+ return serializeToolFailure(error)
+ }
+
logAPICallErrorDiagnostics(error)
console.error('Ephemeral stream response error:', error)
return serializePublicError(error)
diff --git a/lib/tools/__tests__/fetch-error.test.ts b/lib/tools/__tests__/fetch-error.test.ts
new file mode 100644
index 00000000..56e0808c
--- /dev/null
+++ b/lib/tools/__tests__/fetch-error.test.ts
@@ -0,0 +1,41 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import { isToolFailureError } from '@/lib/errors/tool-error'
+import { fetchTool } from '@/lib/tools/fetch'
+
+describe('fetch tool errors', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.restoreAllMocks()
+ })
+
+ it('marks a failed page request as a tool failure', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(
+ async () => new Response(null, { status: 403, statusText: 'Forbidden' })
+ )
+ )
+
+ const result = fetchTool.execute?.(
+ { url: 'https://example.com/private', type: 'regular' },
+ { toolCallId: 'fetch', messages: [] }
+ )
+ expect(result).toBeDefined()
+
+ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()
+ await iterator.next()
+
+ try {
+ await iterator.next()
+ throw new Error('Expected fetch tool to throw')
+ } catch (error) {
+ expect(isToolFailureError(error)).toBe(true)
+ if (isToolFailureError(error)) {
+ expect(error.toolName).toBe('fetch')
+ expect(error.originalMessage).toBe('HTTP 403: Forbidden')
+ }
+ }
+ })
+})
diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.ts
index 9bc16eee..a402b2ae 100644
--- a/lib/tools/fetch.ts
+++ b/lib/tools/fetch.ts
@@ -1,5 +1,6 @@
import { tool, UIToolInvocation } from 'ai'
+import { ToolFailureError } from '@/lib/errors/tool-error'
import { fetchSchema } from '@/lib/schema/fetch'
import { SearchResults as SearchResultsType } from '@/lib/types'
import { logToolPayload } from '@/lib/utils/usage-logging'
@@ -171,17 +172,23 @@ export const fetchTool = tool({
let results: SearchResultsType
- if (type === 'regular') {
- // Use regular fetch for direct HTML retrieval
- results = await fetchRegularData(url)
- } else {
- // Use API-based extraction (Jina or Tavily)
- const useJina = process.env.JINA_API_KEY
- if (useJina) {
- results = await fetchJinaReaderData(url)
+ // Only the retrieval is wrapped: a yield rejects when the consumer stops
+ // reading, and an aborted stream is not a failure of the page.
+ try {
+ if (type === 'regular') {
+ // Use regular fetch for direct HTML retrieval
+ results = await fetchRegularData(url)
} else {
- results = await fetchTavilyExtractData(url)
+ // Use API-based extraction (Jina or Tavily)
+ const useJina = process.env.JINA_API_KEY
+ if (useJina) {
+ results = await fetchJinaReaderData(url)
+ } else {
+ results = await fetchTavilyExtractData(url)
+ }
}
+ } catch (error) {
+ throw new ToolFailureError('fetch', error)
}
logToolPayload('fetch', url, { results: results.results })
diff --git a/lib/tools/search.ts b/lib/tools/search.ts
index 5a088aeb..c2b6b6c7 100644
--- a/lib/tools/search.ts
+++ b/lib/tools/search.ts
@@ -1,5 +1,6 @@
import { type JSONValue, tool, UIToolInvocation } from 'ai'
+import { ToolFailureError } from '@/lib/errors/tool-error'
import { getSearchSchemaForModel } from '@/lib/schema/search'
import { SearchResults } from '@/lib/types'
import {
@@ -139,7 +140,7 @@ export function createSearchTool(fullModel: string) {
} catch (error) {
console.error('Search API error:', error)
// Re-throw the error to let AI SDK handle it properly
- throw error instanceof Error ? error : new Error('Unknown search error')
+ throw new ToolFailureError('search', error)
}
// No citationMap is attached: it fully duplicated `results`