Skip to content

An AI-powered search engine with a generative UI

feat(fetch): option to send regular fetches through the extraction service (#1096)

Yoshiki Miurab7461d4c90bd
diff --git a/.env.local.example b/.env.local.example
index 998ef057..00d22177 100644
--- a/.env.local.example
+++ b/.env.local.example
@@ -228,6 +228,12 @@ ANONYMOUS_USER_ID=anonymous-user
# not reachable from the internet.
# FETCH_ALLOW_PRIVATE_NETWORK=true
+# Send the fetch tool's "regular" requests through the extraction service
+# (Jina or Tavily) instead of requesting the page from this server. Useful while
+# the server's runtime cannot be trusted to decode responses from arbitrary
+# hosts.
+# FETCH_REGULAR_VIA_API=true
+
# Where source favicons come from. The server fetches them and serves them from
# your own origin, so visitors never contact the provider themselves. Use
# {domain} and {size} as placeholders, or set it to "off" to show letter
diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.md
index e5712ba9..700ebe59 100644
--- a/docs/CONFIGURATION.md
+++ b/docs/CONFIGURATION.md
@@ -421,6 +421,12 @@ FETCH_ALLOW_PRIVATE_NETWORK=true
Leave this off on any instance others can reach. With it on, whoever can send a prompt can read whatever the host can reach.
+To keep the server from requesting pages itself, send the `regular` fetch type through the extraction service (Jina Reader if `JINA_API_KEY` is set, otherwise Tavily Extract):
+
+```bash
+FETCH_REGULAR_VIA_API=true
+```
+
### Source Favicons
Favicons for cited sources are fetched by the server and served from your own origin at `/api/favicon`, so a visitor's browser never tells the icon provider which sources a result cites. Responses are cached in memory and at the edge.
diff --git a/lib/tools/__tests__/fetch.test.ts b/lib/tools/__tests__/fetch.test.ts
index 325c88e9..e5eb30d5 100644
--- a/lib/tools/__tests__/fetch.test.ts
+++ b/lib/tools/__tests__/fetch.test.ts
@@ -424,6 +424,33 @@ describe('crawl4ai extraction', () => {
})
})
+ it('sends regular fetches to Tavily Extract with FETCH_REGULAR_VIA_API', async () => {
+ type FetchLike = (url: string | URL | Request) => Promise<Response>
+ const fetchMock = vi.fn<FetchLike>(
+ async () =>
+ new Response(
+ JSON.stringify({ results: [{ url, raw_content: 'Extracted' }] })
+ )
+ )
+ vi.stubGlobal('fetch', fetchMock)
+ vi.stubEnv('TAVILY_API_KEY', 'test-tavily-key')
+ vi.stubEnv('FETCH_REGULAR_VIA_API', 'true')
+
+ const result = fetchTool.execute?.(
+ { url, type: 'regular' },
+ { toolCallId: 'fetch', messages: [], context: {} }
+ )
+ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()
+ await iterator.next()
+ const complete = await iterator.next()
+
+ expect(fetchMock.mock.calls).toHaveLength(1)
+ expect(fetchMock.mock.calls[0]?.[0]).toBe('https://api.tavily.com/extract')
+ expect(complete.value).toMatchObject({
+ results: [{ content: 'Extracted', url }]
+ })
+ })
+
it('rejects an unsuccessful crawl4ai response', async () => {
vi.spyOn(console, 'error').mockImplementation(() => {})
fetchCrawl4ai(
diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.ts
index a9aca0d0..d47dd6fe 100644
--- a/lib/tools/fetch.ts
+++ b/lib/tools/fetch.ts
@@ -379,7 +379,10 @@ export function createFetchTool(options?: {
// crawler's to guard.
await assertResolvedPublicUrl(url)
results = await fetchCrawl4aiData(url)
- } else if (type === 'regular') {
+ } else if (
+ type === 'regular' &&
+ process.env.FETCH_REGULAR_VIA_API !== 'true'
+ ) {
// Use regular fetch for direct HTML retrieval
results = await fetchRegularData(url)
} else {