Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): reject private targets before proxy dispatch in safeFetch (#4864)
Gergő Móricz8838973b4059
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 394be5b68..8db3b596b 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -233,6 +233,11 @@ jobs:run: pnpm exec vitest run src/services/autumn/__tests__/autumn.service.test.ts --reporter=defaultworking-directory: apps/api+ - name: Run safeFetch SSRF guard regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: pnpm exec vitest run src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts --reporter=default+ working-directory: apps/api+- name: Run browser session DELETE regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=defaultdiff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.tsnew file mode 100644index 000000000..bbc914a07--- /dev/null+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts@@ -0,0 +1,269 @@+import { once } from "node:events";+import { createServer, type Server } from "node:http";+import { connect, type Socket } from "node:net";+import { Agent, fetch, interceptors, request, type Dispatcher } from "undici";+import {+ afterAll,+ afterEach,+ beforeAll,+ describe,+ expect,+ it,+ vi,+} from "vitest";++const config = vi.hoisted(() => ({+ ALLOW_LOCAL_WEBHOOKS: false as boolean | undefined,+ PROXY_SERVER: undefined as string | undefined,+ PROXY_USERNAME: undefined as string | undefined,+ PROXY_PASSWORD: undefined as string | undefined,+}));++vi.mock("../../../../config", () => ({ config }));++import {+ InsecureConnectionError,+ rejectPrivateIPLiteralTargets,+} from "./safeFetch";++const previousAllowLocalWebhooks = config.ALLOW_LOCAL_WEBHOOKS;++afterEach(() => {+ config.ALLOW_LOCAL_WEBHOOKS = previousAllowLocalWebhooks;+});++function invoke(origin: string) {+ const inner = vi.fn(() => true) as unknown as Dispatcher.Dispatch;+ const onError = vi.fn();+ const dispatch = rejectPrivateIPLiteralTargets(inner);++ const accepted = dispatch(+ {+ origin,+ path: "/",+ method: "GET",+ },+ { onError } as unknown as Dispatcher.DispatchHandler,+ );++ return { accepted, inner, onError };+}++async function listen(server: Server, host?: string) {+ server.listen(0, host);+ await once(server, "listening");++ const address = server.address();+ if (!address || typeof address === "string") {+ throw new Error("Expected a TCP test server");+ }++ return address.port;+}++async function close(server: Server) {+ server.closeAllConnections();+ server.close();+ await once(server, "close");+}++function createTarget() {+ const target = { hits: 0, server: createServer() };+ target.server.on("request", (_req, res) => {+ target.hits += 1;+ res.end("private target reached");+ });+ return target;+}++function createConnectProxy() {+ const proxy = { connections: 0, server: createServer() };+ const tunnels = new Set<Socket>();+ proxy.server.on("connection", () => {+ proxy.connections += 1;+ });+ proxy.server.on("connect", (req, clientSocket: Socket, head) => {+ const [host, port] = (req.url ?? "").split(":");+ const upstream = connect(Number(port), host, () => {+ clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");+ upstream.write(head);+ upstream.pipe(clientSocket);+ clientSocket.pipe(upstream);+ });+ tunnels.add(clientSocket).add(upstream);+ upstream.on("error", () => clientSocket.destroy());+ clientSocket.on("error", () => upstream.destroy());+ });+ const closeProxy = async () => {+ tunnels.forEach(socket => socket.destroy());+ await close(proxy.server);+ };+ return Object.assign(proxy, { close: closeProxy });+}++describe("proxy destination IP-literal guard", () => {+ it("rejects an IPv4 loopback target before dispatch", () => {+ config.ALLOW_LOCAL_WEBHOOKS = false;++ const { accepted, inner, onError } = invoke("http://127.0.0.1:9911");++ expect(accepted).toBe(true);+ expect(inner).not.toHaveBeenCalled();+ expect(onError).toHaveBeenCalledOnce();+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);+ });++ it("rejects a bracketed IPv6 loopback target before dispatch", () => {+ config.ALLOW_LOCAL_WEBHOOKS = false;++ const { inner, onError } = invoke("http://[::1]:9911");++ expect(inner).not.toHaveBeenCalled();+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);+ });++ it("passes a public IP literal to the underlying dispatcher", () => {+ config.ALLOW_LOCAL_WEBHOOKS = false;++ const { inner, onError } = invoke("https://93.184.216.34/");++ expect(inner).toHaveBeenCalledOnce();+ expect(onError).not.toHaveBeenCalled();+ });++ it("preserves the explicit local-network opt-in", () => {+ config.ALLOW_LOCAL_WEBHOOKS = true;++ const { inner, onError } = invoke("http://127.0.0.1:9911");++ expect(inner).toHaveBeenCalledOnce();+ expect(onError).not.toHaveBeenCalled();+ });++ it.each([+ "http://2130706433",+ "http://0x7f.0.0.1",+ "http://0177.0.0.1",+ "http://127.1",+ "http://127.0.0.1.",+ "http://0",+ "http://10.0.0.1",+ "http://169.254.169.254",+ "http://[::ffff:127.0.0.1]",+ "http://[fc00::1]",+ "http://localhost",+ "http://LOCALHOST.",+ "http://api.localhost",+ ])("rejects the private target %s before dispatch", origin => {+ config.ALLOW_LOCAL_WEBHOOKS = false;++ const { inner, onError } = invoke(origin);++ expect(inner).not.toHaveBeenCalled();+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);+ });++ it.each([+ "https://example.com",+ "http://localhost.example.com",+ "https://[2606:4700:4700::1111]",+ ])("passes the public target %s to the underlying dispatcher", origin => {+ config.ALLOW_LOCAL_WEBHOOKS = false;++ const { inner, onError } = invoke(origin);++ expect(inner).toHaveBeenCalledOnce();+ expect(onError).not.toHaveBeenCalled();+ });++ it("re-applies the guard when the redirect interceptor follows a Location", async () => {+ config.ALLOW_LOCAL_WEBHOOKS = false;+ let redirectorHits = 0;++ const privateTarget = createTarget();+ const privateTargetPort = await listen(privateTarget.server, "127.0.0.1");++ const redirector = createServer((_req, res) => {+ redirectorHits += 1;+ res.writeHead(302, {+ location: `http://127.0.0.1:${privateTargetPort}/secret`,+ });+ res.end();+ });+ const redirectorPort = await listen(redirector, "127.0.0.1");++ const dispatcher = new Agent({+ connect: {+ lookup: (_hostname, options, callback) =>+ options.all+ ? callback(null, [{ address: "127.0.0.1", family: 4 }])+ : callback(null, "127.0.0.1", 4),+ },+ }).compose(+ rejectPrivateIPLiteralTargets,+ interceptors.redirect({ maxRedirections: 5 }),+ );++ try {+ await expect(+ request(`http://redirector.test:${redirectorPort}/start`, {+ dispatcher,+ }),+ ).rejects.toBeInstanceOf(InsecureConnectionError);+ expect(redirectorHits).toBe(1);+ expect(privateTarget.hits).toBe(0);+ } finally {+ await dispatcher.close();+ await close(redirector);+ await close(privateTarget.server);+ }+ });+});++describe("getSecureDispatcher with PROXY_SERVER configured", () => {+ const target = createTarget();+ const proxy = createConnectProxy();+ let targetUrl: string;+ let safeFetch: typeof import("./safeFetch.js");++ beforeAll(async () => {+ targetUrl = `http://127.0.0.1:${await listen(target.server, "127.0.0.1")}/secret`;+ config.PROXY_SERVER = `http://127.0.0.1:${await listen(proxy.server, "127.0.0.1")}`;+ vi.resetModules();+ safeFetch = await import("./safeFetch.js");+ });++ afterAll(async () => {+ config.PROXY_SERVER = undefined;+ await proxy.close();+ await close(target.server);+ });++ it("never opens a proxy tunnel to a private IP literal", async () => {+ config.ALLOW_LOCAL_WEBHOOKS = false;+ const proxyConnections = proxy.connections;+ const targetHits = target.hits;++ const error = await fetch(targetUrl, {+ dispatcher: safeFetch.getSecureDispatcher(),+ }).catch(e => e);++ expect(error).toBeInstanceOf(TypeError);+ expect(error.cause).toBeInstanceOf(safeFetch.InsecureConnectionError);+ expect(proxy.connections).toBe(proxyConnections);+ expect(target.hits).toBe(targetHits);+ });++ it("reaches a local target through the proxy when ALLOW_LOCAL_WEBHOOKS is set", async () => {+ config.ALLOW_LOCAL_WEBHOOKS = true;+ const proxyConnections = proxy.connections;++ const response = await fetch(targetUrl, {+ dispatcher: safeFetch.getSecureDispatcher(),+ });++ expect(response.status).toBe(200);+ expect(await response.text()).toBe("private target reached");+ expect(proxy.connections).toBeGreaterThan(proxyConnections);+ });+});diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.tsindex 4ef7ca8ad..ee535a782 100644--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts@@ -19,6 +19,52 @@ export function isIPPrivate(address: string): boolean {return addr.range() !== "unicast";}+/**+ * Reject private IP literals and localhost names before dispatch, on the+ * initial request and every redirect hop. With PROXY_SERVER set, the socket+ * check below never sees the destination, so this is the only local guard.+ * Other hostnames are resolved proxy-side and must be filtered by the proxy.+ */+export const rejectPrivateIPLiteralTargets: undici.Dispatcher.DispatcherComposeInterceptor =+ dispatch => (options, handler) => {+ if (config.ALLOW_LOCAL_WEBHOOKS === true || options.origin === undefined) {+ return dispatch(options, handler);+ }++ let privateTarget = false;++ try {+ const origin =+ options.origin instanceof URL+ ? options.origin+ : new URL(options.origin.toString());+ const hostname = origin.hostname.replace(/^\[|\]$/g, "");+ privateTarget =+ isIPPrivate(hostname) || /(^|\.)localhost\.?$/.test(hostname);+ } catch {+ // Let Undici report malformed origins through its normal path.+ }++ if (privateTarget) {+ const error = new InsecureConnectionError();+ const compatibleHandler = handler as typeof handler & {+ onResponseError?: (controller: unknown, error: Error) => void;+ };++ if (typeof compatibleHandler.onError === "function") {+ compatibleHandler.onError(error);+ } else if (typeof compatibleHandler.onResponseError === "function") {+ compatibleHandler.onResponseError(null, error);+ } else {+ throw error;+ }++ return true;+ }++ return dispatch(options, handler);+ };+function createBaseAgent(skipTlsVerification: boolean) {const baseAgent = config.PROXY_SERVER? new undici.ProxyAgent({@@ -39,7 +85,10 @@ function createBaseAgent(skipTlsVerification: boolean) {});// Add redirect interceptor for handling redirects- return baseAgent.compose(interceptors.redirect({ maxRedirections: 5000 }));+ return baseAgent.compose(+ rejectPrivateIPLiteralTargets,+ interceptors.redirect({ maxRedirections: 5000 }),+ );}function attachSecurityCheck(agent: undici.Dispatcher) {