An AI-powered search engine with a generative UI
TypeScript generative-aigenerative-uinextjsollamareactsearxngshadcn-uitailwindcsstavilytypescriptupstashvercel-ai-sdk
fix(fetch): refuse private network addresses in fetch and crawl (#1008)
Yoshiki Miura9560e88c227c
diff --git a/.env.local.example b/.env.local.exampleindex ca6b7f49..ac57e7c5 100644--- a/.env.local.example+++ b/.env.local.example@@ -186,5 +186,12 @@ ANONYMOUS_USER_ID=anonymous-user# -----------------------------------------------------------------------------# Advanced# -----------------------------------------------------------------------------+# Let the fetch tool and the advanced-search crawler reach private address+# space (loopback, RFC1918, link-local). Off by default: with it on, anyone who+# can send a prompt can read services on the host's network, including the+# cloud metadata endpoint. Only turn it on for a single-user instance that is+# not reachable from the internet.+# FETCH_ALLOW_PRIVATE_NETWORK=true+# Feedback Notifications (Slack)# SLACK_WEBHOOK_URL=[YOUR_SLACK_WEBHOOK_URL]diff --git a/app/api/advanced-search/route.ts b/app/api/advanced-search/route.tsindex 90dedf46..bfa4facf 100644--- a/app/api/advanced-search/route.ts+++ b/app/api/advanced-search/route.ts@@ -13,6 +13,7 @@ import {SearXNGResult,SearXNGSearchResults} from '@/lib/types'+import { safeFetch } from '@/lib/utils/safe-fetch'/*** Maximum number of results to fetch from SearXNG.@@ -322,9 +323,10 @@ async function crawlPage(virtualConsole.on('error', () => {})virtualConsole.on('warn', () => {})+ // No `resources` option: a crawled page decides its own subresource URLs,+ // so loading them would let it aim the server wherever it likes.const dom = new JSDOM(html, {runScripts: 'outside-only',- resources: 'usable',virtualConsole})const document = dom.window.document@@ -599,40 +601,16 @@ async function fetchHtmlWithTimeout(}}-function fetchHtml(url: string): Promise<string> {- return new Promise((resolve, reject) => {- const protocol = url.startsWith('https:') ? https : http- const agent = url.startsWith('https:') ? httpsAgent : httpAgent- const request = protocol.get(url, { agent }, res => {- if (- res.statusCode &&- res.statusCode >= 300 &&- res.statusCode < 400 &&- res.headers.location- ) {- // Handle redirects- fetchHtml(new URL(res.headers.location, url).toString())- .then(resolve)- .catch(reject)- return- }- let data = ''- res.on('data', chunk => {- data += chunk- })- res.on('end', () => resolve(data))- })- request.on('error', error => {- //console.error(`Error fetching ${url}:`, error)- reject(error)- })- request.on('timeout', () => {- request.destroy()- //reject(new Error(`Request timed out for ${url}`))- resolve('')- })- request.setTimeout(10000) // 10 second timeout- })+async function fetchHtml(url: string): Promise<string> {+ const controller = new AbortController()+ const timeoutId = setTimeout(() => controller.abort(), 10000)++ try {+ const response = await safeFetch(url, { signal: controller.signal })+ return await response.text()+ } finally {+ clearTimeout(timeoutId)+ }}function timeout(ms: number, message: string): Promise<never> {diff --git a/bun.lock b/bun.lockindex e6013db9..24fc4fd9 100644--- a/bun.lock+++ b/bun.lock@@ -80,6 +80,7 @@"streamdown": "^2.5.0","tailwind-merge": "^2.6.0","tailwindcss-animate": "^1.0.7",+ "undici": "^7","vaul": "^1.1.2","zod": "^4.3.6",},diff --git a/docs/CONFIGURATION.md b/docs/CONFIGURATION.mdindex e219d72d..d33ec53d 100644--- a/docs/CONFIGURATION.md+++ b/docs/CONFIGURATION.md@@ -198,6 +198,8 @@ ANONYMOUS_USER_ID=anonymous-userAll users share a single anonymous user ID. No additional configuration needed.+Anyone who can reach the instance can send prompts, and a prompt is what decides which addresses the server requests on the sender's behalf.+### Enabling Supabase AuthenticationFor multi-user deployments:@@ -268,6 +270,16 @@ LANGFUSE_PUBLIC_KEY=[YOUR_PUBLIC_KEY]LANGFUSE_BASE_URL=https://cloud.langfuse.com```+### Outbound Fetch++The `fetch` tool and the advanced-search crawler refuse private address space: loopback, RFC1918, link-local (which carries the cloud metadata endpoint), and the other non-global ranges. To let them read a service on the host's own network:++```bash+FETCH_ALLOW_PRIVATE_NETWORK=true+```++Leave this off on any instance others can reach. With it on, whoever can send a prompt can read whatever the host can reach.+### File UploadEnable file upload with Cloudflare R2 or S3-compatible storage:diff --git a/lib/errors/tool-error.ts b/lib/errors/tool-error.tsindex d8b08fec..b48ed815 100644--- a/lib/errors/tool-error.ts+++ b/lib/errors/tool-error.ts@@ -10,6 +10,14 @@ export type ToolName = 'fetch' | 'search'*/export const INVALID_URL_SENTINEL = 'Invalid fetch URL.'+/**+ * Raised when a URL resolves into private address space. Kept apart from the+ * invalid-URL case because the address was well formed: the refusal is about+ * where it points, and the copy below must not repeat that back to the caller,+ * which would turn a refusal into a probe of the internal network.+ */+export const BLOCKED_URL_SENTINEL = 'Blocked fetch URL.'+// What the failure happened to. The whole point of this module is that it is// never the user and never the AI service.const SUBJECT: Record<ToolName, string> = {@@ -71,6 +79,7 @@ const UNAVAILABLE = (subject: string) =>`${subject} is temporarily unavailable. Please try again shortly.`const TOO_SLOW = (subject: string) => `${subject} took too long to respond.`const INVALID_URL = 'The link is not a valid web address.'+const BLOCKED_URL = 'The link points to an address that cannot be read.'const UNREADABLE_TYPE = 'The page is not in a readable text format.'const NO_CONTENT = 'The page did not return any readable content.'const FETCH_FALLBACK = 'The page could not be read.'@@ -93,6 +102,7 @@ export const TOOL_FAILURE_MESSAGES: ReadonlySet<string> = new Set([TOO_SLOW(subject)]),INVALID_URL,+ BLOCKED_URL,UNREADABLE_TYPE,NO_CONTENT,FETCH_FALLBACK,@@ -115,6 +125,10 @@ function classifyToolFailure(error: ToolFailureError): {return { message: INVALID_URL, retryable: false }}+ if (message === BLOCKED_URL_SENTINEL) {+ return { message: BLOCKED_URL, retryable: false }+ }+if (status === 403 || /\b403\b|\bforbidden\b/i.test(message)) {return { message: REFUSED(subject), retryable: false }}diff --git a/lib/tools/__tests__/fetch-ssrf.test.ts b/lib/tools/__tests__/fetch-ssrf.test.tsnew file mode 100644index 00000000..51d7dd7e--- /dev/null+++ b/lib/tools/__tests__/fetch-ssrf.test.ts@@ -0,0 +1,114 @@+// @vitest-environment node+import http from 'node:http'+import type { AddressInfo } from 'node:net'+import { afterEach, describe, expect, it, vi } from 'vitest'++import { isToolFailureError } from '@/lib/errors/tool-error'+import { fetchTool } from '@/lib/tools/fetch'++const CREDENTIALS = '{"AccessKeyId":"AKIAEXAMPLE","SecretAccessKey":"s3cret"}'++function run(url: string, type: 'regular' | 'api') {+ const result = fetchTool.execute?.(+ { url, type },+ { toolCallId: 'fetch', messages: [], context: {} }+ )+ return (result as AsyncIterable<unknown>)[Symbol.asyncIterator]()+}++async function expectBlocked(url: string, type: 'regular' | 'api' = 'regular') {+ const iterator = run(url, type)++ try {+ // The refusal lands before the first yield when the URL alone settles it,+ // and after it when the address is only known once the socket opens.+ await iterator.next()+ await iterator.next()+ throw new Error('Expected fetch tool to throw')+ } catch (error) {+ expect(isToolFailureError(error)).toBe(true)+ if (isToolFailureError(error)) {+ expect(error.originalMessage).toBe('Blocked fetch URL.')+ }+ }+}++/**+ * A stand-in metadata endpoint, reached by name so that neither the literal+ * check nor the resolving one is what refuses it: only the connect-time lookup+ * inside the agent can.+ */+async function withPrivateHost(+ body: (port: number, hits: () => number) => Promise<void>+) {+ let hits = 0+ const server = http.createServer((_, res) => {+ hits += 1+ res.setHeader('content-type', 'application/json')+ res.end(CREDENTIALS)+ })+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))+ const { port } = server.address() as AddressInfo++ try {+ await body(port, () => hits)+ } finally {+ await new Promise<void>(resolve => {+ server.close(() => resolve())+ })+ }+}++describe('fetch tool SSRF guard', () => {+ afterEach(() => {+ vi.unstubAllEnvs()+ vi.unstubAllGlobals()+ vi.restoreAllMocks()+ })++ it('refuses a private literal without a network call', async () => {+ const fetchMock = vi.fn()+ vi.stubGlobal('fetch', fetchMock)++ await expectBlocked('http://169.254.169.254/latest/meta-data/')++ expect(fetchMock).not.toHaveBeenCalled()+ })++ // The extraction service does the requesting, so the name has to be settled+ // before it is handed over rather than at connect time.+ it('refuses a name that resolves into private space on the API path', async () => {+ const fetchMock = vi.fn()+ vi.stubGlobal('fetch', fetchMock)+ vi.stubEnv('JINA_API_KEY', 'test-key')++ await expectBlocked('http://localhost/admin', 'api')++ expect(fetchMock).not.toHaveBeenCalled()+ })++ it('never reaches a live private host, and never returns its body', async () => {+ vi.spyOn(console, 'error').mockImplementation(() => {})++ await withPrivateHost(async (port, hits) => {+ await expectBlocked(`http://localhost:${port}/creds`)+ expect(hits()).toBe(0)+ })+ })++ // The control: without it, a guard that refuses everything would look the+ // same as one that refuses the right thing.+ it('reads the same host once the network is opted in', async () => {+ vi.stubEnv('FETCH_ALLOW_PRIVATE_NETWORK', 'true')++ await withPrivateHost(async (port, hits) => {+ const iterator = run(`http://localhost:${port}/creds`, 'regular')+ await iterator.next()+ const result = await iterator.next()++ expect(result.value).toMatchObject({ state: 'complete' })+ expect(JSON.stringify(result.value)).toContain('SecretAccessKey')+ expect(hits()).toBe(1)+ })+ })+})diff --git a/lib/tools/fetch.ts b/lib/tools/fetch.tsindex 5e198960..0fb8190b 100644--- a/lib/tools/fetch.ts+++ b/lib/tools/fetch.ts@@ -3,6 +3,11 @@ import { tool, UIToolInvocation } from 'ai'import { INVALID_URL_SENTINEL, ToolFailureError } from '@/lib/errors/tool-error'import { fetchSchema } from '@/lib/schema/fetch'import { SearchResults as SearchResultsType } from '@/lib/types'+import {+ assertPublicUrl,+ assertResolvedPublicUrl,+ safeFetch+} from '@/lib/utils/safe-fetch'import { logToolPayload } from '@/lib/utils/usage-logging'const CONTENT_CHARACTER_LIMIT = 50000@@ -23,7 +28,7 @@ async function fetchRegularData(url: string): Promise<SearchResultsType> {const controller = new AbortController()const timeoutId = setTimeout(() => controller.abort(), 10000) // 10 second timeout- const response = await fetch(url, {+ const response = await safeFetch(url, {signal: controller.signal,headers: {'User-Agent': 'Mozilla/5.0 (compatible; Morphic/1.0)',@@ -183,6 +188,13 @@ export const fetchTool = tool({inputSchema: fetchSchema,async *execute({ url, type = 'regular' }) {assertFetchableUrl(url)+ // Ahead of the branch below, so an address the server must not reach is+ // refused whether it would be read here or handed to an extraction service.+ try {+ assertPublicUrl(url)+ } catch (error) {+ throw new ToolFailureError('fetch', error)+ }// Yield initial fetching stateyield {@@ -199,6 +211,10 @@ export const fetchTool = tool({// Use regular fetch for direct HTML retrievalresults = await fetchRegularData(url)} else {+ // The extraction service does the requesting here, so the name has to+ // be settled before it is handed over rather than at connect time.+ await assertResolvedPublicUrl(url)+// Use API-based extraction (Jina or Tavily)const useJina = process.env.JINA_API_KEYif (useJina) {diff --git a/lib/types/ai.ts b/lib/types/ai.tsindex 7fdc1b31..7d9341f1 100644--- a/lib/types/ai.ts+++ b/lib/types/ai.ts@@ -1,9 +1,9 @@import type { ReasoningPart, TextPart } from '@ai-sdk/provider-utils'import type { InferUITool, UIMessage as AIMessage } from 'ai'-import { fetchTool } from '@/lib/tools/fetch'-import { askQuestionTool } from '@/lib/tools/question'-import { searchTool } from '@/lib/tools/search'+import type { fetchTool } from '@/lib/tools/fetch'+import type { askQuestionTool } from '@/lib/tools/question'+import type { searchTool } from '@/lib/tools/search'import { createTodoTools, type TodoItem } from '@/lib/tools/todo'import type { SearchMode } from '@/lib/types/search'diff --git a/lib/utils/__tests__/safe-fetch-lookup.test.ts b/lib/utils/__tests__/safe-fetch-lookup.test.tsnew file mode 100644index 00000000..87955021--- /dev/null+++ b/lib/utils/__tests__/safe-fetch-lookup.test.ts@@ -0,0 +1,87 @@+// @vitest-environment node+import type { LookupAddress } from 'node:dns'+import { beforeEach, describe, expect, it, vi } from 'vitest'++const dnsLookup = vi.fn()++vi.mock('node:dns', () => ({+ lookup: (+ hostname: string,+ options: unknown,+ callback: (error: unknown, addresses: unknown) => void+ ) => dnsLookup(hostname, options, callback)+}))++const { guardedLookup } = await import('@/lib/utils/safe-fetch')++function resolveTo(addresses: LookupAddress[]) {+ dnsLookup.mockImplementation((_hostname, _options, callback) => {+ callback(null, addresses)+ })+}++function call(options: Record<string, unknown>) {+ return new Promise<{+ error: unknown+ address: string | LookupAddress[]+ family?: number+ }>(resolve => {+ guardedLookup('example.com', options, (error, address, family) =>+ resolve({ error, address, family })+ )+ })+}++describe('guardedLookup callback contract', () => {+ beforeEach(() => {+ dnsLookup.mockReset()+ })++ /**+ * Family autoselection is what `net.connect` turns on by default, and it+ * hands back the whole list. Answering it with a single string aborts the+ * connection, which would take down every allowed host rather than a+ * blocked one.+ */+ it('answers an all-address lookup with the list', async () => {+ const addresses = [+ { address: '93.184.216.34', family: 4 },+ { address: '2606:2800:220:1:248:1893:25c8:1946', family: 6 }+ ]+ resolveTo(addresses)++ const result = await call({ all: true })++ expect(result.error).toBeNull()+ expect(result.address).toEqual(addresses)+ })++ it('answers a single-address lookup with the address and family', async () => {+ resolveTo([{ address: '93.184.216.34', family: 4 }])++ const result = await call({})++ expect(result.error).toBeNull()+ expect(result.address).toBe('93.184.216.34')+ expect(result.family).toBe(4)+ })++ it('refuses the host when any answer is private', async () => {+ resolveTo([+ { address: '93.184.216.34', family: 4 },+ { address: '127.0.0.1', family: 4 }+ ])++ const result = await call({ all: true })++ expect(result.error).toBeInstanceOf(Error)+ })++ it('refuses an empty answer', async () => {+ resolveTo([])++ const result = await call({ all: true })++ expect(result.error).toBeInstanceOf(Error)+ })+})diff --git a/lib/utils/__tests__/safe-fetch.test.ts b/lib/utils/__tests__/safe-fetch.test.tsnew file mode 100644index 00000000..8fd43080--- /dev/null+++ b/lib/utils/__tests__/safe-fetch.test.ts@@ -0,0 +1,176 @@+// @vitest-environment node+import http from 'node:http'+import type { AddressInfo } from 'node:net'+import { afterEach, describe, expect, it, vi } from 'vitest'++import { BLOCKED_URL_SENTINEL } from '@/lib/errors/tool-error'+import {+ assertPublicUrl,+ isBlockedAddress,+ safeFetch+} from '@/lib/utils/safe-fetch'++describe('isBlockedAddress', () => {+ it.each([+ '127.0.0.1',+ '127.1.2.3',+ '0.0.0.0',+ '10.1.2.3',+ '172.16.0.1',+ '172.31.255.255',+ '192.168.1.1',+ '169.254.169.254',+ '100.64.0.1',+ '255.255.255.255',+ '::1',+ '::',+ 'fe80::1',+ 'fd00::1',+ 'fec0::1',+ 'ff02::1',+ // The metadata address wearing each of its IPv6 disguises.+ '::ffff:169.254.169.254',+ '::ffff:a9fe:a9fe',+ // ::ffff:0:0:0/96, the translatable prefix, one group over from the mapped+ // form above.+ '::ffff:0:a9fe:a9fe',+ '::ffff:0:127.0.0.1',+ '64:ff9b::169.254.169.254',+ // RFC 8215 local-use NAT64, which embeds IPv4 at a different offset.+ '64:ff9b:1::169.254.169.254',+ '64:ff9b:1:ffff::1',+ // Reserved ranges a host network can still route somewhere internal.+ '192.0.2.1',+ '198.51.100.1',+ '203.0.113.1',+ '192.88.99.1',+ '2001:db8::1'+ ])('blocks %s', address => {+ expect(isBlockedAddress(address)).toBe(true)+ })++ it.each([+ '1.1.1.8',+ '8.8.8.8',+ '93.184.216.34',+ '172.32.0.1',+ '2606:4700::1111'+ ])('allows %s', address => {+ expect(isBlockedAddress(address)).toBe(false)+ })++ it('blocks anything that is not an address', () => {+ expect(isBlockedAddress('not-an-address')).toBe(true)+ })+})++describe('assertPublicUrl', () => {+ it('rejects a private literal', () => {+ expect(() =>+ assertPublicUrl('http://169.254.169.254/latest/meta-data/')+ ).toThrow(BLOCKED_URL_SENTINEL)+ expect(() => assertPublicUrl('http://[::1]:8080/')).toThrow(+ BLOCKED_URL_SENTINEL+ )+ })++ it('rejects a non-http scheme', () => {+ expect(() => assertPublicUrl('file:///etc/passwd')).toThrow(+ BLOCKED_URL_SENTINEL+ )+ })++ it('accepts a public URL', () => {+ expect(() => assertPublicUrl('https://example.com/page')).not.toThrow()+ })++ it('accepts a private literal once the network is opted in', () => {+ vi.stubEnv('FETCH_ALLOW_PRIVATE_NETWORK', 'true')+ expect(() => assertPublicUrl('http://127.0.0.1:3000/')).not.toThrow()+ vi.unstubAllEnvs()+ })+})++describe('safeFetch against a live loopback server', () => {+ afterEach(() => {+ vi.unstubAllEnvs()+ vi.unstubAllGlobals()+ })++ async function withServer<T>(+ run: (port: number, hits: () => number) => Promise<T>+ ) {+ let hits = 0+ const server = http.createServer((_, res) => {+ hits += 1+ res.end('secret')+ })+ await new Promise<void>(resolve => server.listen(0, '127.0.0.1', resolve))+ const { port } = server.address() as AddressInfo++ try {+ return await run(port, () => hits)+ } finally {+ await new Promise<void>(resolve => {+ server.close(() => resolve())+ })+ }+ }++ // `localhost` is a name, so nothing but the connect-time lookup can catch it.+ it('refuses a hostname that resolves to loopback, without connecting', async () => {+ await withServer(async (port, hits) => {+ await expect(safeFetch(`http://localhost:${port}/`)).rejects.toThrow()+ expect(hits()).toBe(0)+ })+ })++ it('reaches the same server once the network is opted in', async () => {+ vi.stubEnv('FETCH_ALLOW_PRIVATE_NETWORK', 'true')++ await withServer(async (port, hits) => {+ const response = await safeFetch(`http://localhost:${port}/`)+ expect(await response.text()).toBe('secret')+ expect(hits()).toBe(1)+ })+ })+})++describe('safeFetch redirects', () => {+ afterEach(() => {+ vi.unstubAllGlobals()+ })++ it('re-checks every hop', async () => {+ vi.stubGlobal(+ 'fetch',+ vi.fn(+ async () =>+ new Response(null, {+ status: 302,+ headers: { location: 'http://169.254.169.254/latest/meta-data/' }+ })+ )+ )++ await expect(safeFetch('https://example.com/start')).rejects.toThrow(+ BLOCKED_URL_SENTINEL+ )+ })++ it('gives up on a chain that never lands', async () => {+ const fetchMock = vi.fn(+ async () =>+ new Response(null, {+ status: 302,+ headers: { location: 'https://example.com/next' }+ })+ )+ vi.stubGlobal('fetch', fetchMock)++ await expect(safeFetch('https://example.com/start')).rejects.toThrow(+ 'Too many redirects'+ )+ expect(fetchMock).toHaveBeenCalledTimes(6)+ })+})diff --git a/lib/utils/safe-fetch.ts b/lib/utils/safe-fetch.tsnew file mode 100644index 00000000..d4d10b39--- /dev/null+++ b/lib/utils/safe-fetch.ts@@ -0,0 +1,305 @@+import {+ lookup as dnsLookup,+ type LookupAddress,+ type LookupOptions+} from 'node:dns'+import { lookup as dnsLookupAsync } from 'node:dns/promises'+import { isIP, type LookupFunction } from 'node:net'+import { Agent } from 'undici'++import { BLOCKED_URL_SENTINEL } from '@/lib/errors/tool-error'++const MAX_REDIRECTS = 5+const REDIRECT_STATUSES = new Set([301, 302, 303, 307, 308])++// Ranges that never belong to a public host: loopback, the RFC1918 space, the+// link-local block that carries cloud metadata, carrier NAT, and the reserved+// and multicast tails.+const BLOCKED_IPV4_RANGES: ReadonlyArray<readonly [string, number]> = [+ ['0.0.0.0', 8],+ ['10.0.0.0', 8],+ ['100.64.0.0', 10],+ ['127.0.0.0', 8],+ ['169.254.0.0', 16],+ ['172.16.0.0', 12],+ ['192.0.0.0', 24],+ ['192.0.2.0', 24],+ ['192.88.99.0', 24],+ ['192.168.0.0', 16],+ ['198.18.0.0', 15],+ ['198.51.100.0', 24],+ ['203.0.113.0', 24],+ ['224.0.0.0', 4],+ ['240.0.0.0', 4]+]++function allowsPrivateNetwork(): boolean {+ return process.env.FETCH_ALLOW_PRIVATE_NETWORK === 'true'+}++function parseIpv4(value: string): number | null {+ const parts = value.split('.')+ if (parts.length !== 4) return null++ let result = 0+ for (const part of parts) {+ if (!/^\d{1,3}$/.test(part)) return null+ const octet = Number(part)+ if (octet > 255) return null+ result = result * 256 + octet+ }+ return result+}++function isBlockedIpv4(address: number): boolean {+ return BLOCKED_IPV4_RANGES.some(([network, bits]) => {+ const base = parseIpv4(network)+ if (base === null) return false+ const mask = bits === 0 ? 0 : (-1 << (32 - bits)) >>> 0+ return (address & mask) >>> 0 === (base & mask) >>> 0+ })+}++// Returns the eight 16-bit groups, so a compressed address and its embedded+// IPv4 tail are compared in the same shape as a fully written one.+function expandIpv6(value: string): number[] | null {+ let address = value.split('%')[0]++ if (address.includes('.')) {+ const lastColon = address.lastIndexOf(':')+ if (lastColon === -1) return null+ const embedded = parseIpv4(address.slice(lastColon + 1))+ if (embedded === null) return null+ const high = (embedded >>> 16).toString(16)+ const low = (embedded & 0xffff).toString(16)+ address = `${address.slice(0, lastColon + 1)}${high}:${low}`+ }++ const halves = address.split('::')+ if (halves.length > 2) return null++ const toGroups = (part: string) =>+ part.length === 0 ? [] : part.split(':').map(g => Number.parseInt(g, 16))++ const head = toGroups(halves[0])+ const tail = halves.length === 2 ? toGroups(halves[1]) : []+ if (+ [...head, ...tail].some(g => !Number.isInteger(g) || g < 0 || g > 0xffff)+ ) {+ return null+ }++ if (halves.length === 1) return head.length === 8 ? head : null++ const filler = 8 - head.length - tail.length+ if (filler < 0) return null+ return [...head, ...Array(filler).fill(0), ...tail]+}++function isBlockedIpv6(value: string): boolean {+ const groups = expandIpv6(value)+ if (groups === null) return true++ const [g0, g1, g2, g3, g4, g5, g6, g7] = groups+ const embeddedV4 = (((g6 << 16) | g7) >>> 0) as number+ const leadingZeros = g0 === 0 && g1 === 0 && g2 === 0 && g3 === 0 && g4 === 0++ // ::ffff:a.b.c.d, ::a.b.c.d and the NAT64 prefix all reach an IPv4 host, so+ // they are judged as that host rather than as an IPv6 one.+ if (leadingZeros && g5 === 0xffff) return isBlockedIpv4(embeddedV4)+ if (leadingZeros && g5 === 0) {+ // Covers :: and ::1 as well, both of which fall inside 0.0.0.0/8.+ return embeddedV4 <= 1 || isBlockedIpv4(embeddedV4)+ }+ // ::ffff:0:0:0/96, which SIIT translates to the address in the last 32 bits.+ // One group over from the mapped form above, and a different prefix.+ if (+ g0 === 0 &&+ g1 === 0 &&+ g2 === 0 &&+ g3 === 0 &&+ g4 === 0xffff &&+ g5 === 0+ ) {+ return isBlockedIpv4(embeddedV4)+ }+ // The whole NAT64 space rather than the well-known prefix alone: RFC 6052+ // embeds the IPv4 address at an offset that depends on the prefix length, so+ // reading it back is only right for the lengths we thought to handle. A+ // translation prefix is never a destination worth reaching on its own.+ if (g0 === 0x0064 && g1 === 0xff9b) return true++ if (g0 === 0x2001 && g1 === 0x0db8) return true // 2001:db8::/32, documentation+ if ((g0 & 0xfe00) === 0xfc00) return true // fc00::/7, unique local+ if ((g0 & 0xffc0) === 0xfe80) return true // fe80::/10, link local+ if ((g0 & 0xffc0) === 0xfec0) return true // fec0::/10, site local+ if ((g0 & 0xff00) === 0xff00) return true // ff00::/8, multicast+ return false+}++export function isBlockedAddress(address: string): boolean {+ const family = isIP(address)+ if (family === 4) {+ const parsed = parseIpv4(address)+ return parsed === null || isBlockedIpv4(parsed)+ }+ if (family === 6) return isBlockedIpv6(address)+ return true+}++/**+ * A `net.connect` lookup that resolves the name itself and refuses the whole+ * host when any answer is private. Deciding at connect time is what closes the+ * gap a separate pre-flight check leaves open: a name can answer twice, once+ * for the check and once for the socket.+ */+export const guardedLookup: LookupFunction = (hostname, options, callback) => {+ dnsLookup(hostname, { ...options, all: true }, (error, addresses) => {+ if (error) {+ callback(error, '', 0)+ return+ }++ const resolved = addresses as LookupAddress[]+ if (+ resolved.length === 0 ||+ resolved.some(a => isBlockedAddress(a.address))+ ) {+ callback(new Error(BLOCKED_URL_SENTINEL), '', 0)+ return+ }++ // Family autoselection asks for every address and expects the list back.+ // Answering with a single string there aborts the connection instead of+ // allowing it, which fails every name the guard was meant to let through.+ if (options.all === true) {+ callback(null, resolved)+ return+ }++ callback(null, resolved[0].address, resolved[0].family)+ })+}++let guardedAgent: Agent | undefined++function getGuardedAgent(): Agent {+ guardedAgent ??= new Agent({ connect: { lookup: guardedLookup } })+ return guardedAgent+}++/**+ * Rejects a URL that names private address space by literal. A hostname is not+ * settled here, only at connect time, so this is the pre-flight for callers+ * that hand the URL to something other than `safeFetch`.+ */+export function assertPublicUrl(value: string): void {+ let parsed: URL+ try {+ parsed = new URL(value)+ } catch {+ throw new Error(BLOCKED_URL_SENTINEL)+ }++ if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') {+ throw new Error(BLOCKED_URL_SENTINEL)+ }+ if (allowsPrivateNetwork()) return++ // A literal is settled here; a name is settled by the lookup above.+ const hostname = parsed.hostname.replace(/^\[|\]$/g, '')+ if (isIP(hostname) !== 0 && isBlockedAddress(hostname)) {+ throw new Error(BLOCKED_URL_SENTINEL)+ }+}++/**+ * The resolving counterpart of `assertPublicUrl`, for a URL that is about to+ * be handed to an extraction service rather than requested here. Those+ * requests leave from the service, so this is not about reaching our own+ * network: it is about not making Morphic the thing that points a third party+ * at private space.+ */+export async function assertResolvedPublicUrl(value: string): Promise<void> {+ assertPublicUrl(value)+ if (allowsPrivateNetwork()) return++ const hostname = new URL(value).hostname.replace(/^\[|\]$/g, '')+ if (isIP(hostname) !== 0) return++ let addresses: LookupAddress[]+ try {+ addresses = await dnsLookupAsync(hostname, { all: true })+ } catch {+ throw new Error(BLOCKED_URL_SENTINEL)+ }++ if (+ addresses.length === 0 ||+ addresses.some(a => isBlockedAddress(a.address))+ ) {+ throw new Error(BLOCKED_URL_SENTINEL)+ }+}++interface SafeFetchInit {+ headers?: Record<string, string>+ signal?: AbortSignal+}++// `dispatcher` is undici's, which the platform fetch reads but does not+// declare.+type DispatchedInit = RequestInit & { dispatcher?: Agent }++/**+ * The refusal raised inside the lookup reaches the caller wrapped, with the+ * sentinel buried in the cause chain, where the failure classifier cannot see+ * it and reports a generic retryable error instead.+ */+function isBlockedError(error: unknown): boolean {+ let current: unknown = error++ for (let depth = 0; depth < 5; depth++) {+ if (!(current instanceof Error)) return false+ if (current.message === BLOCKED_URL_SENTINEL) return true+ current = current.cause+ }+ return false+}++/**+ * Fetch that refuses private address space, on the first request and on every+ * redirect it is asked to follow. Redirects are taken by hand because the+ * automatic ones are followed inside the platform, where the destination is+ * never offered for inspection.+ */+export async function safeFetch(+ url: string,+ init: SafeFetchInit = {}+): Promise<Response> {+ let target = url++ for (let hop = 0; hop <= MAX_REDIRECTS; hop++) {+ assertPublicUrl(target)++ let response: Response+ try {+ response = await fetch(target, {+ ...init,+ redirect: 'manual',+ ...(allowsPrivateNetwork() ? {} : { dispatcher: getGuardedAgent() })+ } as DispatchedInit)+ } catch (error) {+ if (isBlockedError(error)) throw new Error(BLOCKED_URL_SENTINEL)+ throw error+ }++ const location = response.headers.get('location')+ if (!REDIRECT_STATUSES.has(response.status) || !location) return response++ await response.body?.cancel().catch(() => {})+ target = new URL(location, target).toString()+ }++ throw new Error(`Too many redirects: ${url}`)+}diff --git a/package.json b/package.jsonindex 53bf2341..ce2204b8 100644--- a/package.json+++ b/package.json@@ -94,6 +94,7 @@"streamdown": "^2.5.0","tailwind-merge": "^2.6.0","tailwindcss-animate": "^1.0.7",+ "undici": "^7","vaul": "^1.1.2","zod": "^4.3.6"},