An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(errors): describe a failed tool call by its resource, not by the app (#949)
Yoshiki Miurabab0eb2a0c05
diff --git a/lib/errors/__tests__/public-error.test.ts b/lib/errors/__tests__/public-error.test.tsindex f5c3a39c..bf6c628c 100644--- a/lib/errors/__tests__/public-error.test.ts+++ b/lib/errors/__tests__/public-error.test.ts@@ -6,6 +6,7 @@ import {serializePublicError,toPublicErrorPayload} from '@/lib/errors/public-error'+import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'describe('public error mapping', () => {it('hides provider billing errors', () => {@@ -171,6 +172,42 @@ describe('public error mapping', () => {expect(serialized).not.toContain('Redis timeout')})+ it('preserves serialized tool failure copy when parsing it again', () => {+ const serialized = serializeToolFailure(+ new ToolFailureError('fetch', 'HTTP 403: Forbidden')+ )+ const payload = toPublicErrorPayload(serialized)++ expect(payload.code).toBe('tool_failed')+ expect(payload.type).toBe('general')+ expect(payload.error).toBe('The page refused the request.')+ })++ it('does not preserve upstream copy that only claims the tool_failed code', () => {+ const error = new Error(+ JSON.stringify({+ code: 'tool_failed',+ error: 'connection to db-primary failed for user svc_admin'+ })+ )+ const payload = toPublicErrorPayload(error)++ expect(payload.error).not.toContain('svc_admin')+ expect(payload.error).not.toContain('db-primary')+ })++ it('keeps request-level 403 failures on the permission path', () => {+ const error = Object.assign(new Error('Request failed'), {+ statusCode: 403+ })+ const payload = toPublicErrorPayload(error)++ expect(payload.code).toBe('forbidden')+ expect(payload.error).toBe(+ 'You do not have permission to access this resource.'+ )+ })+it('creates JSON error responses without raw messages', async () => {const response = createPublicErrorResponse(new Error(diff --git a/lib/errors/__tests__/tool-error.test.ts b/lib/errors/__tests__/tool-error.test.tsnew file mode 100644index 00000000..b7261ecd--- /dev/null+++ b/lib/errors/__tests__/tool-error.test.ts@@ -0,0 +1,130 @@+import { describe, expect, it } from 'vitest'++import {+ getToolFailureMessage,+ isToolFailureError,+ isToolFailureMessage,+ serializeToolFailure,+ ToolFailureError+} from '@/lib/errors/tool-error'++describe('tool error mapping', () => {+ it.each([+ ['HTTP 403: Forbidden', 'The page refused the request.'],+ ['Forbidden by upstream', 'The page refused the request.'],+ ['HTTP 404: Not Found', 'The page could not be found.'],+ [+ 'HTTP 429: Too Many Requests',+ 'The page is rate limiting requests. Please try again shortly.'+ ],+ [+ 'HTTP 500: Internal Server Error',+ 'The page is temporarily unavailable. Please try again shortly.'+ ],+ [+ 'Service unavailable',+ 'The page is temporarily unavailable. Please try again shortly.'+ ],+ ['Request timeout after 10 seconds', 'The page took too long to respond.'],+ ['The request was aborted', 'The page took too long to respond.'],+ [+ 'Unsupported content type: application/pdf',+ 'The page is not in a readable text format.'+ ],+ [+ 'No results returned from content extraction service',+ 'The page did not return any readable content.'+ ],+ [+ 'No data returned from Jina Reader API',+ 'The page did not return any readable content.'+ ]+ ])('maps %s to scoped copy', (message, expected) => {+ expect(getToolFailureMessage(new ToolFailureError('fetch', message))).toBe(+ expected+ )+ })++ it('uses a fetch-specific unknown fallback', () => {+ expect(+ getToolFailureMessage(new ToolFailureError('fetch', 'Unexpected failure'))+ ).toBe('The page could not be read.')+ })++ it('uses a search-specific unknown fallback', () => {+ expect(+ getToolFailureMessage(+ new ToolFailureError('search', 'Unexpected failure')+ )+ ).toBe('The search could not be completed.')+ })++ it('recognizes markers without relying only on instanceof', () => {+ expect(+ isToolFailureError({+ isToolFailureError: true,+ toolName: 'fetch',+ originalMessage: 'failed'+ })+ ).toBe(true)+ expect(isToolFailureError(new Error('failed'))).toBe(false)+ })++ it('scopes the copy to the search service', () => {+ expect(+ getToolFailureMessage(+ new ToolFailureError('search', 'HTTP 403: Forbidden')+ )+ ).toBe('The search service refused the request.')+ })++ it('does not mark a refusal as retryable', () => {+ const payload = JSON.parse(+ serializeToolFailure(new ToolFailureError('fetch', 'HTTP 403: Forbidden'))+ )++ expect(payload.retryable).toBe(false)+ })++ it('serializes a tool_failed public payload', () => {+ const payload = JSON.parse(+ serializeToolFailure(new ToolFailureError('search', 'Unexpected failure'))+ )++ expect(payload).toMatchObject({+ code: 'tool_failed',+ type: 'general',+ error: 'The search could not be completed.'+ })+ })+})++describe('tool failure copy', () => {+ it('only recognizes sentences the mapper can produce', () => {+ expect(isToolFailureMessage('The page refused the request.')).toBe(true)+ expect(isToolFailureMessage('Something else entirely.')).toBe(false)+ })++ it.each([+ ['fetch', 'HTTP 403: Forbidden'],+ ['fetch', 'HTTP 404: Not Found'],+ ['fetch', 'HTTP 429: Too Many Requests'],+ ['fetch', 'HTTP 503: Service Temporarily Unavailable'],+ ['fetch', 'Request timeout after 10 seconds'],+ ['fetch', 'Unsupported content type: application/pdf'],+ ['fetch', 'No results returned from content extraction service'],+ ['fetch', 'Unexpected failure'],+ ['search', 'HTTP 403: Forbidden'],+ ['search', 'HTTP 429: Too Many Requests'],+ ['search', 'Unexpected failure']+ ] as const)(+ 'keeps %s copy for %s inside the known set',+ (toolName, message) => {+ expect(+ isToolFailureMessage(+ getToolFailureMessage(new ToolFailureError(toolName, message))+ )+ ).toBe(true)+ }+ )+})diff --git a/lib/errors/public-error.ts b/lib/errors/public-error.tsindex 139203b2..1f2ed3a9 100644--- a/lib/errors/public-error.ts+++ b/lib/errors/public-error.ts@@ -1,3 +1,5 @@+import { isToolFailureMessage } from './tool-error'+export type PublicErrorType = 'rate-limit' | 'auth' | 'forbidden' | 'general'export type PublicErrorCode =@@ -12,6 +14,7 @@ export type PublicErrorCode =| 'provider_rate_limit'| 'provider_unavailable'| 'rate_limit'+ | 'tool_failed'| 'unknown'export type PublicErrorPayload = {@@ -54,6 +57,7 @@ const PUBLIC_ERROR_CODES: ReadonlySet<string> = new Set(['provider_rate_limit','provider_unavailable','rate_limit',+ 'tool_failed','unknown'])@@ -371,6 +375,16 @@ function classifyError(snapshot: ErrorSnapshot, fallbackMessage?: string) {}}+/**+ * A `tool_failed` code alone is not proof of provenance: an upstream error+ * whose own text happens to be JSON reaches this same parser, and honouring+ * the code would hand that text straight to the client. Only copy this module+ * knows it produced is preserved.+ */+function isToolFailurePayload(message: string, code: PublicErrorCode): boolean {+ return code === 'tool_failed' && isToolFailureMessage(message)+}+function isIntentionalPublicPayload(value: Record<string, unknown>,code: PublicErrorCode@@ -402,7 +416,9 @@ function fromParsedPayload()const code = asPublicErrorCode(value.code) ?? classification.codeconst type = asPublicErrorType(value.type) ?? typeForCode(code)- const preserveMessage = isIntentionalPublicPayload(value, code)+ const preserveMessage =+ isIntentionalPublicPayload(value, code) ||+ isToolFailurePayload(message, code)return {error: preserveMessage ? message : classification.error,diff --git a/lib/errors/tool-error.ts b/lib/errors/tool-error.tsnew file mode 100644index 00000000..5f2ba42f--- /dev/null+++ b/lib/errors/tool-error.ts@@ -0,0 +1,161 @@+import type { PublicErrorPayload } from './public-error'++export type ToolName = 'fetch' | 'search'++// What the failure happened to. The whole point of this module is that it is+// never the user and never the AI service.+const SUBJECT: Record<ToolName, string> = {+ fetch: 'The page',+ search: 'The search service'+}++function getErrorMessage(error: unknown): string {+ return error instanceof Error ? error.message : String(error)+}++function getErrorStatus(error: unknown): number | undefined {+ if (typeof error !== 'object' || error === null) return undefined++ const record = error as Record<string, unknown>+ const status = record.statusCode ?? record.status+ return typeof status === 'number' && Number.isFinite(status)+ ? status+ : undefined+}++export class ToolFailureError extends Error {+ readonly isToolFailureError = true+ readonly originalMessage: string+ readonly status?: number++ constructor(+ readonly toolName: ToolName,+ error: unknown+ ) {+ const originalMessage = getErrorMessage(error)+ super(originalMessage, { cause: error })+ this.name = 'ToolFailureError'+ this.originalMessage = originalMessage+ this.status = getErrorStatus(error)+ }+}++/**+ * Structural rather than `instanceof`, so a failure that crossed a bundle+ * boundary is still recognized as one.+ */+export function isToolFailureError(error: unknown): error is ToolFailureError {+ if (typeof error !== 'object' || error === null) return false++ const value = error as Record<string, unknown>+ return (+ value.isToolFailureError === true &&+ (value.toolName === 'fetch' || value.toolName === 'search') &&+ typeof value.originalMessage === 'string'+ )+}++const REFUSED = (subject: string) => `${subject} refused the request.`+const NOT_FOUND = (subject: string) => `${subject} could not be found.`+const RATE_LIMITED = (subject: string) =>+ `${subject} is rate limiting requests. Please try again shortly.`+const UNAVAILABLE = (subject: string) =>+ `${subject} is temporarily unavailable. Please try again shortly.`+const TOO_SLOW = (subject: string) => `${subject} took too long to respond.`+const UNREADABLE_TYPE = 'The page is not in a readable text format.'+const NO_CONTENT = 'The page did not return any readable content.'+const FETCH_FALLBACK = 'The page could not be read.'+const SEARCH_FALLBACK = 'The search could not be completed.'++/**+ * Every sentence this module can produce.+ *+ * `public-error` preserves a `tool_failed` message instead of reclassifying it,+ * and an error whose own text happens to be JSON is parsed by that same code+ * path. Checking membership here means an upstream payload claiming the code+ * can still only carry copy we wrote ourselves.+ */+export const TOOL_FAILURE_MESSAGES: ReadonlySet<string> = new Set([+ ...Object.values(SUBJECT).flatMap(subject => [+ REFUSED(subject),+ NOT_FOUND(subject),+ RATE_LIMITED(subject),+ UNAVAILABLE(subject),+ TOO_SLOW(subject)+ ]),+ UNREADABLE_TYPE,+ NO_CONTENT,+ FETCH_FALLBACK,+ SEARCH_FALLBACK+])++export function isToolFailureMessage(message: string): boolean {+ return TOOL_FAILURE_MESSAGES.has(message)+}++function classifyToolFailure(error: ToolFailureError): {+ message: string+ retryable: boolean+} {+ const message = error.originalMessage+ const status = error.status+ const subject = SUBJECT[error.toolName]++ if (status === 403 || /\b403\b|\bforbidden\b/i.test(message)) {+ return { message: REFUSED(subject), retryable: false }+ }++ if (status === 404 || /\b404\b|\bnot found\b/i.test(message)) {+ return { message: NOT_FOUND(subject), retryable: false }+ }++ if (+ status === 429 ||+ /\b429\b|too many requests|rate[_\s-]?limit/i.test(message)+ ) {+ return { message: RATE_LIMITED(subject), retryable: true }+ }++ if (+ (status !== undefined && status >= 500 && status <= 599) ||+ /\b5\d{2}\b|service unavailable|bad gateway/i.test(message)+ ) {+ return { message: UNAVAILABLE(subject), retryable: true }+ }++ if (/timeout|timed out|abort/i.test(message)) {+ return { message: TOO_SLOW(subject), retryable: true }+ }++ if (/unsupported content type/i.test(message)) {+ return { message: UNREADABLE_TYPE, retryable: false }+ }++ if (+ /no results returned from content extraction service/i.test(message) ||+ /no data returned from jina reader api/i.test(message)+ ) {+ return { message: NO_CONTENT, retryable: true }+ }++ return {+ message: error.toolName === 'fetch' ? FETCH_FALLBACK : SEARCH_FALLBACK,+ retryable: true+ }+}++export function getToolFailureMessage(error: ToolFailureError): string {+ return classifyToolFailure(error).message+}++export function serializeToolFailure(error: ToolFailureError): string {+ const { message, retryable } = classifyToolFailure(error)+ const payload: PublicErrorPayload = {+ error: message,+ code: 'tool_failed',+ type: 'general',+ retryable+ }++ return JSON.stringify(payload)+}diff --git a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.tsindex c84fe08e..3788ad14 100644--- a/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts+++ b/lib/streaming/__tests__/create-ephemeral-chat-stream-response.test.ts@@ -47,7 +47,7 @@ vi.mock('@/lib/utils/usage-logging', () => ({logUsage: vi.fn()}))-import { serializePublicError } from '@/lib/errors/public-error'+import { serializeToolFailure, ToolFailureError } from '@/lib/errors/tool-error'import { createEphemeralChatStreamResponse } from '@/lib/streaming/create-ephemeral-chat-stream-response'import { describeStreamError } from '@/lib/streaming/helpers/describe-stream-error'import { EMPTY_RESPONSE_STATUS_MESSAGE } from '@/lib/streaming/helpers/is-empty-response'@@ -133,15 +133,16 @@ describe('createEphemeralChatStreamResponse', () => {})it('does not mark the span as failed for a tool error', async () => {- const toolError = Object.assign(new Error('HTTP 403: Forbidden'), {- statusCode: 403- })+ const toolError = new ToolFailureError(+ 'fetch',+ new Error('HTTP 403: Forbidden')+ )mocks.stream.mockResolvedValue(createFakeResult({ toolError }))await createEphemeralChatStreamResponse(createConfig())await mocks.finishPromise- expect(mocks.serializedError).toBe(serializePublicError(toolError))+ expect(mocks.serializedError).toBe(serializeToolFailure(toolError))expect(mocks.span.update).not.toHaveBeenCalled()expect(mocks.span.end).toHaveBeenCalledOnce()expect(mocks.forceFlush).toHaveBeenCalledOnce()diff --git a/lib/streaming/create-chat-stream-response.ts b/lib/streaming/create-chat-stream-response.tsindex 36bdaae1..72cfc304 100644--- a/lib/streaming/create-chat-stream-response.ts+++ b/lib/streaming/create-chat-stream-response.ts@@ -8,6 +8,10 @@ import {createPublicErrorResponse,serializePublicError} from '@/lib/errors/public-error'+import {+ isToolFailureError,+ serializeToolFailure+} from '@/lib/errors/tool-error'import { isTracingEnabled } from '@/lib/utils/telemetry'import { loadChat } from '../actions/chat'@@ -256,6 +260,11 @@ export async function createChatStreamResponse(}},onError: (error: unknown) => {+ if (isToolFailureError(error)) {+ console.error('Tool failure:', error)+ return serializeToolFailure(error)+ }+logAPICallErrorDiagnostics(error)console.error('Stream response error:', error)return serializePublicError(error)diff --git a/lib/streaming/create-ephemeral-chat-stream-response.ts b/lib/streaming/create-ephemeral-chat-stream-response.tsindex 2b601d1c..9d90e527 100644--- a/lib/streaming/create-ephemeral-chat-stream-response.ts+++ b/lib/streaming/create-ephemeral-chat-stream-response.ts@@ -8,6 +8,10 @@ import {createPublicErrorResponse,serializePublicError} from '@/lib/errors/public-error'+import {+ isToolFailureError,+ serializeToolFailure+} from '@/lib/errors/tool-error'import { isTracingEnabled } from '@/lib/utils/telemetry'import {@@ -154,6 +158,11 @@ export async function createEphemeralChatStreamResponse(await endTracing()},onError: (error: unknown) => {+ if (isToolFailureError(error)) {+ console.error('Ephemeral tool failure:', error)+ return serializeToolFailure(error)+ }+logAPICallErrorDiagnostics(error)console.error('Ephemeral stream response error:', error)return serializePublicError(error)diff --git a/lib/tools/__tests__/fetch-error.test.ts b/lib/tools/__tests__/fetch-error.test.tsnew file mode 100644index 00000000..56e0808c--- /dev/null+++ b/lib/tools/__tests__/fetch-error.test.ts@@ -0,0 +1,41 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++import { isToolFailureError } from '@/lib/errors/tool-error'+import { fetchTool } from '@/lib/tools/fetch'++describe('fetch tool errors', () => {+ afterEach(() => {+ vi.unstubAllGlobals()+ vi.restoreAllMocks()+ })++ it('marks a failed page request as a tool failure', async () => {+ vi.spyOn(console, 'error').mockImplementation(() => {})+ vi.stubGlobal(+ 'fetch',+ vi.fn(+ async () => new Response(null, { status: 403, statusText: 'Forbidden' })+ )+ )++ const result = fetchTool.execute?.(+ { url: 'https://example.com/private', type: 'regular' },+ { toolCallId: 'fetch', messages: [] }+ )+ expect(result).toBeDefined()++ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()+ await iterator.next()++ try {+ await iterator.next()+ throw new Error('Expected fetch tool to throw')+ } catch (error) {+ expect(isToolFailureError(error)).toBe(true)+ if (isToolFailureError(error)) {+ expect(error.toolName).toBe('fetch')+ expect(error.originalMessage).toBe('HTTP 403: Forbidden')+ }+ }+ })+})diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex 9bc16eee..a402b2ae 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -1,5 +1,6 @@import { tool, UIToolInvocation } from 'ai'+import { ToolFailureError } from '@/lib/errors/tool-error'import { fetchSchema } from '@/lib/schema/fetch'import { SearchResults as SearchResultsType } from '@/lib/types'import { logToolPayload } from '@/lib/utils/usage-logging'@@ -171,17 +172,23 @@ export const fetchTool = tool({let results: SearchResultsType- if (type === 'regular') {- // Use regular fetch for direct HTML retrieval- results = await fetchRegularData(url)- } else {- // Use API-based extraction (Jina or Tavily)- const useJina = process.env.JINA_API_KEY- if (useJina) {- results = await fetchJinaReaderData(url)+ // Only the retrieval is wrapped: a yield rejects when the consumer stops+ // reading, and an aborted stream is not a failure of the page.+ try {+ if (type === 'regular') {+ // Use regular fetch for direct HTML retrieval+ results = await fetchRegularData(url)} else {- results = await fetchTavilyExtractData(url)+ // Use API-based extraction (Jina or Tavily)+ const useJina = process.env.JINA_API_KEY+ if (useJina) {+ results = await fetchJinaReaderData(url)+ } else {+ results = await fetchTavilyExtractData(url)+ }}+ } catch (error) {+ throw new ToolFailureError('fetch', error)}logToolPayload('fetch', url, { results: results.results })diff --git a/lib/tools/search.ts b/lib/tools/search.tsindex 5a088aeb..c2b6b6c7 100644--- a/lib/tools/search.ts+++ b/lib/tools/search.ts@@ -1,5 +1,6 @@import { type JSONValue, tool, UIToolInvocation } from 'ai'+import { ToolFailureError } from '@/lib/errors/tool-error'import { getSearchSchemaForModel } from '@/lib/schema/search'import { SearchResults } from '@/lib/types'import {@@ -139,7 +140,7 @@ export function createSearchTool(fullModel: string) {} catch (error) {console.error('Search API error:', error)// Re-throw the error to let AI SDK handle it properly- throw error instanceof Error ? error : new Error('Unknown search error')+ throw new ToolFailureError('search', error)}// No citationMap is attached: it fully duplicated `results`