An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
feat(fetch): option to send regular fetches through the extraction service (#1096)
Yoshiki Miurab7461d4c90bd
diff --git a/.env.local.example b/.env.local.exampleindex 998ef057..00d22177 100644--- a/.env.local.example+++ b/.env.local.example@@ -228,6 +228,12 @@ ANONYMOUS_USER_ID=anonymous-user# not reachable from the internet.# FETCH_ALLOW_PRIVATE_NETWORK=true+# Send the fetch tool's "regular" requests through the extraction service+# (Jina or Tavily) instead of requesting the page from this server. Useful while+# the server's runtime cannot be trusted to decode responses from arbitrary+# hosts.+# FETCH_REGULAR_VIA_API=true+# Where source favicons come from. The server fetches them and serves them from# your own origin, so visitors never contact the provider themselves. Use# {domain} and {size} as placeholders, or set it to "off" to show letterdiff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.mdindex e5712ba9..700ebe59 100644--- a/docs/CONFIGURATION.md+++ b/docs/CONFIGURATION.md@@ -421,6 +421,12 @@ FETCH_ALLOW_PRIVATE_NETWORK=trueLeave this off on any instance others can reach. With it on, whoever can send a prompt can read whatever the host can reach.+To keep the server from requesting pages itself, send the `regular` fetch type through the extraction service (Jina Reader if `JINA_API_KEY` is set, otherwise Tavily Extract):++```bash+FETCH_REGULAR_VIA_API=true+```+### Source FaviconsFavicons for cited sources are fetched by the server and served from your own origin at `/api/favicon`, so a visitor's browser never tells the icon provider which sources a result cites. Responses are cached in memory and at the edge.diff --git a/lib/tools/__tests__/fetch.test.ts b/lib/tools/__tests__/fetch.test.tsindex 325c88e9..e5eb30d5 100644--- a/lib/tools/__tests__/fetch.test.ts+++ b/lib/tools/__tests__/fetch.test.ts@@ -424,6 +424,33 @@ describe('crawl4ai extraction', () => {})})+ it('sends regular fetches to Tavily Extract with FETCH_REGULAR_VIA_API', async () => {+ type FetchLike = (url: string | URL | Request) => Promise<Response>+ const fetchMock = vi.fn<FetchLike>(+ async () =>+ new Response(+ JSON.stringify({ results: [{ url, raw_content: 'Extracted' }] })+ )+ )+ vi.stubGlobal('fetch', fetchMock)+ vi.stubEnv('TAVILY_API_KEY', 'test-tavily-key')+ vi.stubEnv('FETCH_REGULAR_VIA_API', 'true')++ const result = fetchTool.execute?.(+ { url, type: 'regular' },+ { toolCallId: 'fetch', messages: [], context: {} }+ )+ const iterator = (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()+ await iterator.next()+ const complete = await iterator.next()++ expect(fetchMock.mock.calls).toHaveLength(1)+ expect(fetchMock.mock.calls[0]?.[0]).toBe('https://api.tavily.com/extract')+ expect(complete.value).toMatchObject({+ results: [{ content: 'Extracted', url }]+ })+ })+it('rejects an unsuccessful crawl4ai response', async () => {vi.spyOn(console, 'error').mockImplementation(() => {})fetchCrawl4ai(diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex a9aca0d0..d47dd6fe 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -379,7 +379,10 @@ export function createFetchTool(options?: {// crawler's to guard.await assertResolvedPublicUrl(url)results = await fetchCrawl4aiData(url)- } else if (type === 'regular') {+ } else if (+ type === 'regular' &&+ process.env.FETCH_REGULAR_VIA_API !== 'true'+ ) {// Use regular fetch for direct HTML retrievalresults = await fetchRegularData(url)} else {