An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(fetch): reject a tool argument that is not a URL before requesting it (#955)
Yoshiki Miura919cc021e9fb
diff --git a/lib/errors/__tests__/tool-error.test.ts b/lib/errors/__tests__/tool-error.test.tsindex b7261ecd..82189c4a 100644--- a/lib/errors/__tests__/tool-error.test.ts+++ b/lib/errors/__tests__/tool-error.test.ts@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest'import {getToolFailureMessage,+ INVALID_URL_SENTINEL,isToolFailureError,isToolFailureMessage,serializeToolFailure,@@ -9,6 +10,19 @@ import {} from '@/lib/errors/tool-error'describe('tool error mapping', () => {+ it('maps an invalid fetch URL to non-retryable scoped copy', () => {+ const expected = 'The link is not a valid web address.'+ const error = new ToolFailureError('fetch', INVALID_URL_SENTINEL)++ expect(getToolFailureMessage(error)).toBe(expected)+ // Membership is what lets the copy survive the client's re-parse.+ expect(isToolFailureMessage(expected)).toBe(true)+ expect(JSON.parse(serializeToolFailure(error))).toMatchObject({+ error: expected,+ retryable: false+ })+ })+it.each([['HTTP 403: Forbidden', 'The page refused the request.'],['Forbidden by upstream', 'The page refused the request.'],diff --git a/lib/errors/tool-error.ts b/lib/errors/tool-error.tsindex 5f2ba42f..d8b08fec 100644--- a/lib/errors/tool-error.ts+++ b/lib/errors/tool-error.ts@@ -2,6 +2,14 @@ import type { PublicErrorPayload } from './public-error'export type ToolName = 'fetch' | 'search'+/**+ * Raised by the fetch tool when its argument is not an address at all, so the+ * classifier can tell that case apart from a page that really did fail. The+ * platform's own text embeds the rejected value, which would otherwise put a+ * model-authored string into the trace and into the copy shown to the user.+ */+export const INVALID_URL_SENTINEL = 'Invalid fetch URL.'+// What the failure happened to. The whole point of this module is that it is// never the user and never the AI service.const SUBJECT: Record<ToolName, string> = {@@ -62,6 +70,7 @@ const RATE_LIMITED = (subject: string) =>const UNAVAILABLE = (subject: string) =>`${subject} is temporarily unavailable. Please try again shortly.`const TOO_SLOW = (subject: string) => `${subject} took too long to respond.`+const INVALID_URL = 'The link is not a valid web address.'const UNREADABLE_TYPE = 'The page is not in a readable text format.'const NO_CONTENT = 'The page did not return any readable content.'const FETCH_FALLBACK = 'The page could not be read.'@@ -83,6 +92,7 @@ export const TOOL_FAILURE_MESSAGES: ReadonlySet<string> = new Set([UNAVAILABLE(subject),TOO_SLOW(subject)]),+ INVALID_URL,UNREADABLE_TYPE,NO_CONTENT,FETCH_FALLBACK,@@ -101,6 +111,10 @@ function classifyToolFailure(error: ToolFailureError): {const status = error.statusconst subject = SUBJECT[error.toolName]+ if (error.toolName === 'fetch' && message === INVALID_URL_SENTINEL) {+ return { message: INVALID_URL, retryable: false }+ }+if (status === 403 || /\b403\b|\bforbidden\b/i.test(message)) {return { message: REFUSED(subject), retryable: false }}diff --git a/lib/schema/fetch.tsx b/lib/schema/fetch.tsxindex c0d78843..c249fe6c 100644--- a/lib/schema/fetch.tsx+++ b/lib/schema/fetch.tsx@@ -2,7 +2,9 @@ import { DeepPartial } from 'ai'import { z } from 'zod'export const fetchSchema = z.object({- url: z.string().describe('The URL to retrieve content from'),+ url: z+ .string()+ .describe('An absolute http:// or https:// URL to retrieve content from'),type: z.enum(['regular', 'api']).default('regular')diff --git a/lib/tools/__tests__/fetch-error.test.ts b/lib/tools/__tests__/fetch-error.test.tsindex 56e0808c..c5efc568 100644--- a/lib/tools/__tests__/fetch-error.test.ts+++ b/lib/tools/__tests__/fetch-error.test.ts@@ -38,4 +38,65 @@ describe('fetch tool errors', () => {}}})++ it.each([+ 'URL To Web design Template Features',+ 'example.com/page',+ 'file:///etc/passwd',+ // `new URL` reads this as the host `url-to-web-design-template-features`.+ 'https:URL-To-Web-design-Template-Features',+ 'https://'+ ])('rejects invalid URL %s without a network call', async url => {+ const fetchMock = vi.fn()+ vi.stubGlobal('fetch', fetchMock)++ const result = fetchTool.execute?.(+ { url, type: 'regular' },+ { toolCallId: 'fetch', messages: [] }+ )+ expect(result).toBeDefined()++ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()++ try {+ await iterator.next()+ throw new Error('Expected fetch tool to throw')+ } catch (error) {+ expect(isToolFailureError(error)).toBe(true)+ if (isToolFailureError(error)) {+ expect(error.toolName).toBe('fetch')+ expect(error.originalMessage).toBe('Invalid fetch URL.')+ expect(isToolFailureError(error.cause)).toBe(false)+ }+ }++ expect(fetchMock).not.toHaveBeenCalled()+ })++ it('passes a valid HTTPS URL to the network unchanged', async () => {+ const url = 'https://example.com/path?query=value'+ const fetchMock = vi.fn(+ async () =>+ new Response(+ '<html><title>Example</title><body>Content</body></html>',+ {+ headers: { 'content-type': 'text/html' }+ }+ )+ )+ vi.stubGlobal('fetch', fetchMock)++ const result = fetchTool.execute?.(+ { url, type: 'regular' },+ { toolCallId: 'fetch', messages: [] }+ )+ expect(result).toBeDefined()++ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()+ await iterator.next()+ await iterator.next()++ expect(fetchMock).toHaveBeenCalledOnce()+ expect(fetchMock).toHaveBeenCalledWith(url, expect.any(Object))+ })})diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex a402b2ae..29fa8e71 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -1,6 +1,6 @@import { tool, UIToolInvocation } from 'ai'-import { ToolFailureError } from '@/lib/errors/tool-error'+import { INVALID_URL_SENTINEL, ToolFailureError } from '@/lib/errors/tool-error'import { fetchSchema } from '@/lib/schema/fetch'import { SearchResults as SearchResultsType } from '@/lib/types'import { logToolPayload } from '@/lib/utils/usage-logging'@@ -8,6 +8,16 @@ import { logToolPayload } from '@/lib/utils/usage-logging'const CONTENT_CHARACTER_LIMIT = 50000const TITLE_CHARACTER_LIMIT = 100+// The model sometimes fills this argument with a description of what it wants+// rather than an address, so nothing downstream can assume it is one. Parsing+// alone is too weak a test: `new URL` reads `https:some-description` as a host+// named after the description, which would still be looked up.+function assertFetchableUrl(url: string): void {+ if (!/^https?:\/\//i.test(url) || !URL.canParse(url)) {+ throw new ToolFailureError('fetch', INVALID_URL_SENTINEL)+ }+}+async function fetchRegularData(url: string): Promise<SearchResultsType> {try {const controller = new AbortController()@@ -164,6 +174,8 @@ export const fetchTool = tool({'Fetch content from any URL. By default uses "regular" type which performs fast, direct HTML fetching without external APIs - ideal for most websites. IMPORTANT: "regular" type does NOT support PDFs and will fail on PDF URLs. Use "api" type when you need: 1) PDF content extraction (required for .pdf URLs), 2) Complex JavaScript-rendered pages, 3) Better markdown formatting, 4) Table extraction. The "api" type requires Jina or Tavily API keys and uses Jina Reader if available, otherwise falls back to Tavily Extract.',inputSchema: fetchSchema,async *execute({ url, type = 'regular' }) {+ assertFetchableUrl(url)+// Yield initial fetching stateyield {state: 'fetching' as const,