Skip to content

An AI-powered search engine with a generative UI

fix(tools): read JSON and plain text in the regular fetch path (#988)

Yoshiki Miura18ddcd88a6d7
diff --git a/lib/tools/__tests__/fetch.test.ts b/lib/tools/__tests__/fetch.test.ts
new file mode 100644
index 00000000..f953e921
--- /dev/null
+++ b/lib/tools/__tests__/fetch.test.ts
@@ -0,0 +1,74 @@
+import { afterEach, describe, expect, it, vi } from 'vitest'
+
+import { fetchTool } from '@/lib/tools/fetch'
+
+const url = 'https://example.com/resource'
+
+async function fetchRegular(body: string, contentType: string) {
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(
+ async () =>
+ new Response(body, { headers: { 'content-type': contentType } })
+ )
+ )
+
+ const result = fetchTool.execute?.(
+ { url, type: 'regular' },
+ { toolCallId: 'fetch', messages: [], context: {} }
+ )
+ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()
+
+ await iterator.next()
+ return iterator.next()
+}
+
+describe('regular fetch content types', () => {
+ afterEach(() => {
+ vi.unstubAllGlobals()
+ vi.restoreAllMocks()
+ })
+
+ it('returns JSON verbatim', async () => {
+ const body = '{\n "markup": "<value>",\n "spaced": "a b"\n}'
+ const result = await fetchRegular(body, 'application/json; charset=utf-8')
+
+ expect(result.value).toMatchObject({
+ results: [{ content: body, title: 'example.com', url }],
+ state: 'complete'
+ })
+ })
+
+ it('accepts structured JSON media types', async () => {
+ const result = await fetchRegular('{"data":[]}', 'application/vnd.api+json')
+
+ expect(result.value).toMatchObject({ state: 'complete' })
+ })
+
+ it('returns plain text without tag stripping', async () => {
+ const body = 'Keep <literal> tags and whitespace.'
+ const result = await fetchRegular(body, 'text/plain')
+
+ expect(result.value).toMatchObject({ results: [{ content: body }] })
+ })
+
+ it('continues to process HTML content', async () => {
+ const result = await fetchRegular(
+ '<html><title>Page title</title><body>Hello <strong>world</strong><script>bad()</script></body></html>',
+ 'text/html'
+ )
+
+ expect(result.value).toMatchObject({
+ results: [{ content: 'Page title Hello world', title: 'Page title' }]
+ })
+ })
+
+ it('rejects unsupported content types', async () => {
+ vi.spyOn(console, 'error').mockImplementation(() => {})
+ const result = fetchRegular('PDF', 'application/pdf')
+
+ await expect(result).rejects.toThrow(
+ 'Unsupported content type: application/pdf'
+ )
+ })
+})
diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.ts
index 29fa8e71..5e198960 100644
--- a/lib/tools/fetch.ts
+++ b/lib/tools/fetch.ts
@@ -39,39 +39,47 @@ async function fetchRegularData(url: string): Promise<SearchResultsType> {
}
const contentType = response.headers.get('content-type') || ''
- if (
- !contentType.includes('text/html') &&
- !contentType.includes('text/plain')
- ) {
+ // The request above accepts anything, so refusing everything that is not
+ // markup turns readable text into a dead end: a JSON endpoint is the usual
+ // case. Structured bodies take the passthrough below instead of the markup
+ // pipeline, whose tag stripping deletes any run between angle brackets.
+ const mediaType = contentType.split(';', 1)[0].trim().toLowerCase()
+ const isHtmlContent =
+ mediaType === 'text/html' || mediaType === 'application/xhtml+xml'
+ const isTextContent =
+ mediaType === 'text/plain' ||
+ mediaType === 'application/json' ||
+ mediaType.endsWith('+json')
+
+ if (!isHtmlContent && !isTextContent) {
throw new Error(`Unsupported content type: ${contentType}`)
}
- const html = await response.text()
+ const body = await response.text()
- // Extract title
- const titleMatch = html.match(/<title[^>]*>([^<]*)<\/title>/i)
+ const titleMatch = isHtmlContent
+ ? body.match(/<title[^>]*>([^<]*)<\/title>/i)
+ : null
const rawTitle = titleMatch ? titleMatch[1].trim() : new URL(url).hostname
const title =
rawTitle.length > TITLE_CHARACTER_LIMIT
? rawTitle.substring(0, TITLE_CHARACTER_LIMIT) + '...'
: rawTitle
- // Process HTML content
- let processedHtml = html
- .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts
- .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles
-
- // Replace img tags with alt text or [IMAGE] markers
- processedHtml = processedHtml
- .replace(/<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE: $1] ')
- .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')
- .replace(/<img[^>]*>/gi, ' [IMAGE] ')
-
- // Extract text content
- const textContent = processedHtml
- .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags
- .replace(/\s+/g, ' ') // Normalize whitespace
- .trim()
+ const textContent = isHtmlContent
+ ? body
+ .replace(/<script[^>]*>[\s\S]*?<\/script>/gi, '') // Remove scripts
+ .replace(/<style[^>]*>[\s\S]*?<\/style>/gi, '') // Remove styles
+ .replace(
+ /<img[^>]+alt\s*=\s*["']([^"']+)["'][^>]*>/gi,
+ ' [IMAGE: $1] '
+ )
+ .replace(/<img[^>]+src\s*=\s*["']([^"']+)["'][^>]*>/gi, ' [IMAGE] ')
+ .replace(/<img[^>]*>/gi, ' [IMAGE] ')
+ .replace(/<[^>]*>/g, ' ') // Remove remaining HTML tags
+ .replace(/\s+/g, ' ') // Normalize whitespace
+ .trim()
+ : body
// Limit content length
const truncatedContent =