An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(tools): read JSON and plain text in the regular fetch path (#988)
Yoshiki Miura18ddcd88a6d7
diff --git a/lib/tools/__tests__/fetch.test.ts b/lib/tools/__tests__/fetch.test.tsnew file mode 100644index 00000000..f953e921--- /dev/null+++ b/lib/tools/__tests__/fetch.test.ts@@ -0,0 +1,74 @@+import { afterEach, describe, expect, it, vi } from 'vitest'++import { fetchTool } from '@/lib/tools/fetch'++const url = 'https://example.com/resource'++async function fetchRegular(body: string, contentType: string) {+ vi.stubGlobal(+ 'fetch',+ vi.fn(+ async () =>+ new Response(body, { headers: { 'content-type': contentType } })+ )+ )++ const result = fetchTool.execute?.(+ { url, type: 'regular' },+ { toolCallId: 'fetch', messages: [], context: {} }+ )+ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()++ await iterator.next()+ return iterator.next()+}++describe('regular fetch content types', () => {+ afterEach(() => {+ vi.unstubAllGlobals()+ vi.restoreAllMocks()+ })++ it('returns JSON verbatim', async () => {+ const body = '{\n "markup": "<value>",\n "spaced": "a b"\n}'+ const result = await fetchRegular(body, 'application/json; charset=utf-8')++ expect(result.value).toMatchObject({+ results: [{ content: body, title: 'example.com', url }],+ state: 'complete'+ })+ })++ it('accepts structured JSON media types', async () => {+ const result = await fetchRegular('{"data":[]}', 'application/vnd.api+json')++ expect(result.value).toMatchObject({ state: 'complete' })+ })++ it('returns plain text without tag stripping', async () => {+ const body = 'Keep <literal> tags and whitespace.'+ const result = await fetchRegular(body, 'text/plain')++ expect(result.value).toMatchObject({ results: [{ content: body }] })+ })++ it('continues to process HTML content', async () => {+ const result = await fetchRegular(+ '<html><title>Page title</title><body>Hello <strong>world</strong><script>bad()</script></body></html>',+ 'text/html'+ )++ expect(result.value).toMatchObject({+ results: [{ content: 'Page title Hello world', title: 'Page title' }]+ })+ })++ it('rejects unsupported content types', async () => {+ vi.spyOn(console, 'error').mockImplementation(() => {})+ const result = fetchRegular('PDF', 'application/pdf')++ await expect(result).rejects.toThrow(+ 'Unsupported content type: application/pdf'+ )+ })+})diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex 29fa8e71..5e198960 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -39,39 +39,47 @@ async function fetchRegularData(url: string): Promise<SearchResultsType> {}const contentType = response.headers.get('content-type') || ''- if (- !contentType.includes('text/html') &&- !contentType.includes('text/plain')- ) {+ // The request above accepts anything, so refusing everything that is not+ // markup turns readable text into a dead end: a JSON endpoint is the usual+ // case. Structured bodies take the passthrough below instead of the markup+ // pipeline, whose tag stripping deletes any run between angle brackets.+ const mediaType = contentType.split(';', 1)[0].trim().toLowerCase()+ const isHtmlContent =+ mediaType === 'text/html' || mediaType === 'application/xhtml+xml'+ const isTextContent =+ mediaType === 'text/plain' ||+ mediaType === 'application/json' ||+ mediaType.endsWith('+json')++ if (!isHtmlContent && !isTextContent) {throw new Error(`Unsupported content type: ${contentType}`)}- const html = await response.text()+ const body = await response.text()- // Extract title- const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i)+ const titleMatch = isHtmlContent+ ? body.match(/<title[^>]*>([^<]*)<\/title>/i)+ : nullconst rawTitle = titleMatch ? titleMatch[1].trim() : new URL(url).hostnameconst title =rawTitle.length > TITLE_CHARACTER_LIMIT? rawTitle.substring(0, TITLE_CHARACTER_LIMIT) + '...': rawTitle- // Process HTML content- let processedHtml = html- .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts- .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles-- // Replace img tags with alt text or [IMAGE] markers- processedHtml = processedHtml- .replace(/<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE: $1] ')- .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')- .replace(/<img[^>]*>/gi, ' [IMAGE] ')-- // Extract text content- const textContent = processedHtml- .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags- .replace(/\s+/g, ' ') // Normalize whitespace- .trim()+ const textContent = isHtmlContent+ ? body+ .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles+ .replace(+ /<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi,+ ' [IMAGE: $1] '+ )+ .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')+ .replace(/<img[^>]*>/gi, ' [IMAGE] ')+ .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags+ .replace(/\s+/g, ' ') // Normalize whitespace+ .trim()+ : body// Limit content lengthconst truncatedContent =