Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): reject private targets before proxy dispatch in safeFetch (#4864)

Gergő Móricz8838973b4059
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 394be5b68..8db3b596b 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -233,6 +233,11 @@ jobs:
run: pnpm exec vitest run src/services/autumn/__tests__/autumn.service.test.ts --reporter=default
working-directory: apps/api
+ - name: Run safeFetch SSRF guard regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: pnpm exec vitest run src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts --reporter=default
+ working-directory: apps/api
+
- name: Run browser session DELETE regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: pnpm exec vitest run src/controllers/v2/__tests__/browser-delete.test.ts --reporter=default
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts
new file mode 100644
index 000000000..bbc914a07
--- /dev/null
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.proxy.test.ts
@@ -0,0 +1,269 @@
+import { once } from "node:events";
+import { createServer, type Server } from "node:http";
+import { connect, type Socket } from "node:net";
+import { Agent, fetch, interceptors, request, type Dispatcher } from "undici";
+import {
+ afterAll,
+ afterEach,
+ beforeAll,
+ describe,
+ expect,
+ it,
+ vi,
+} from "vitest";
+
+const config = vi.hoisted(() => ({
+ ALLOW_LOCAL_WEBHOOKS: false as boolean | undefined,
+ PROXY_SERVER: undefined as string | undefined,
+ PROXY_USERNAME: undefined as string | undefined,
+ PROXY_PASSWORD: undefined as string | undefined,
+}));
+
+vi.mock("../../../../config", () => ({ config }));
+
+import {
+ InsecureConnectionError,
+ rejectPrivateIPLiteralTargets,
+} from "./safeFetch";
+
+const previousAllowLocalWebhooks = config.ALLOW_LOCAL_WEBHOOKS;
+
+afterEach(() => {
+ config.ALLOW_LOCAL_WEBHOOKS = previousAllowLocalWebhooks;
+});
+
+function invoke(origin: string) {
+ const inner = vi.fn(() => true) as unknown as Dispatcher.Dispatch;
+ const onError = vi.fn();
+ const dispatch = rejectPrivateIPLiteralTargets(inner);
+
+ const accepted = dispatch(
+ {
+ origin,
+ path: "/",
+ method: "GET",
+ },
+ { onError } as unknown as Dispatcher.DispatchHandler,
+ );
+
+ return { accepted, inner, onError };
+}
+
+async function listen(server: Server, host?: string) {
+ server.listen(0, host);
+ await once(server, "listening");
+
+ const address = server.address();
+ if (!address || typeof address === "string") {
+ throw new Error("Expected a TCP test server");
+ }
+
+ return address.port;
+}
+
+async function close(server: Server) {
+ server.closeAllConnections();
+ server.close();
+ await once(server, "close");
+}
+
+function createTarget() {
+ const target = { hits: 0, server: createServer() };
+ target.server.on("request", (_req, res) => {
+ target.hits += 1;
+ res.end("private target reached");
+ });
+ return target;
+}
+
+function createConnectProxy() {
+ const proxy = { connections: 0, server: createServer() };
+ const tunnels = new Set<Socket>();
+ proxy.server.on("connection", () => {
+ proxy.connections += 1;
+ });
+ proxy.server.on("connect", (req, clientSocket: Socket, head) => {
+ const [host, port] = (req.url ?? "").split(":");
+ const upstream = connect(Number(port), host, () => {
+ clientSocket.write("HTTP/1.1 200 Connection Established\r\n\r\n");
+ upstream.write(head);
+ upstream.pipe(clientSocket);
+ clientSocket.pipe(upstream);
+ });
+ tunnels.add(clientSocket).add(upstream);
+ upstream.on("error", () => clientSocket.destroy());
+ clientSocket.on("error", () => upstream.destroy());
+ });
+ const closeProxy = async () => {
+ tunnels.forEach(socket => socket.destroy());
+ await close(proxy.server);
+ };
+ return Object.assign(proxy, { close: closeProxy });
+}
+
+describe("proxy destination IP-literal guard", () => {
+ it("rejects an IPv4 loopback target before dispatch", () => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+
+ const { accepted, inner, onError } = invoke("http://127.0.0.1:9911");
+
+ expect(accepted).toBe(true);
+ expect(inner).not.toHaveBeenCalled();
+ expect(onError).toHaveBeenCalledOnce();
+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);
+ });
+
+ it("rejects a bracketed IPv6 loopback target before dispatch", () => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+
+ const { inner, onError } = invoke("http://[::1]:9911");
+
+ expect(inner).not.toHaveBeenCalled();
+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);
+ });
+
+ it("passes a public IP literal to the underlying dispatcher", () => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+
+ const { inner, onError } = invoke("https://93.184.216.34/");
+
+ expect(inner).toHaveBeenCalledOnce();
+ expect(onError).not.toHaveBeenCalled();
+ });
+
+ it("preserves the explicit local-network opt-in", () => {
+ config.ALLOW_LOCAL_WEBHOOKS = true;
+
+ const { inner, onError } = invoke("http://127.0.0.1:9911");
+
+ expect(inner).toHaveBeenCalledOnce();
+ expect(onError).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ "http://2130706433",
+ "http://0x7f.0.0.1",
+ "http://0177.0.0.1",
+ "http://127.1",
+ "http://127.0.0.1.",
+ "http://0",
+ "http://10.0.0.1",
+ "http://169.254.169.254",
+ "http://[::ffff:127.0.0.1]",
+ "http://[fc00::1]",
+ "http://localhost",
+ "http://LOCALHOST.",
+ "http://api.localhost",
+ ])("rejects the private target %s before dispatch", origin => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+
+ const { inner, onError } = invoke(origin);
+
+ expect(inner).not.toHaveBeenCalled();
+ expect(onError.mock.calls[0][0]).toBeInstanceOf(InsecureConnectionError);
+ });
+
+ it.each([
+ "https://example.com",
+ "http://localhost.example.com",
+ "https://[2606:4700:4700::1111]",
+ ])("passes the public target %s to the underlying dispatcher", origin => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+
+ const { inner, onError } = invoke(origin);
+
+ expect(inner).toHaveBeenCalledOnce();
+ expect(onError).not.toHaveBeenCalled();
+ });
+
+ it("re-applies the guard when the redirect interceptor follows a Location", async () => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+ let redirectorHits = 0;
+
+ const privateTarget = createTarget();
+ const privateTargetPort = await listen(privateTarget.server, "127.0.0.1");
+
+ const redirector = createServer((_req, res) => {
+ redirectorHits += 1;
+ res.writeHead(302, {
+ location: `http://127.0.0.1:${privateTargetPort}/secret`,
+ });
+ res.end();
+ });
+ const redirectorPort = await listen(redirector, "127.0.0.1");
+
+ const dispatcher = new Agent({
+ connect: {
+ lookup: (_hostname, options, callback) =>
+ options.all
+ ? callback(null, [{ address: "127.0.0.1", family: 4 }])
+ : callback(null, "127.0.0.1", 4),
+ },
+ }).compose(
+ rejectPrivateIPLiteralTargets,
+ interceptors.redirect({ maxRedirections: 5 }),
+ );
+
+ try {
+ await expect(
+ request(`http://redirector.test:${redirectorPort}/start`, {
+ dispatcher,
+ }),
+ ).rejects.toBeInstanceOf(InsecureConnectionError);
+ expect(redirectorHits).toBe(1);
+ expect(privateTarget.hits).toBe(0);
+ } finally {
+ await dispatcher.close();
+ await close(redirector);
+ await close(privateTarget.server);
+ }
+ });
+});
+
+describe("getSecureDispatcher with PROXY_SERVER configured", () => {
+ const target = createTarget();
+ const proxy = createConnectProxy();
+ let targetUrl: string;
+ let safeFetch: typeof import("./safeFetch.js");
+
+ beforeAll(async () => {
+ targetUrl = `http://127.0.0.1:${await listen(target.server, "127.0.0.1")}/secret`;
+ config.PROXY_SERVER = `http://127.0.0.1:${await listen(proxy.server, "127.0.0.1")}`;
+ vi.resetModules();
+ safeFetch = await import("./safeFetch.js");
+ });
+
+ afterAll(async () => {
+ config.PROXY_SERVER = undefined;
+ await proxy.close();
+ await close(target.server);
+ });
+
+ it("never opens a proxy tunnel to a private IP literal", async () => {
+ config.ALLOW_LOCAL_WEBHOOKS = false;
+ const proxyConnections = proxy.connections;
+ const targetHits = target.hits;
+
+ const error = await fetch(targetUrl, {
+ dispatcher: safeFetch.getSecureDispatcher(),
+ }).catch(e => e);
+
+ expect(error).toBeInstanceOf(TypeError);
+ expect(error.cause).toBeInstanceOf(safeFetch.InsecureConnectionError);
+ expect(proxy.connections).toBe(proxyConnections);
+ expect(target.hits).toBe(targetHits);
+ });
+
+ it("reaches a local target through the proxy when ALLOW_LOCAL_WEBHOOKS is set", async () => {
+ config.ALLOW_LOCAL_WEBHOOKS = true;
+ const proxyConnections = proxy.connections;
+
+ const response = await fetch(targetUrl, {
+ dispatcher: safeFetch.getSecureDispatcher(),
+ });
+
+ expect(response.status).toBe(200);
+ expect(await response.text()).toBe("private target reached");
+ expect(proxy.connections).toBeGreaterThan(proxyConnections);
+ });
+});
diff --git a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
index 4ef7ca8ad..ee535a782 100644
--- a/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
+++ b/apps/api/src/scraper/scrapeURL/engines/utils/safeFetch.ts
@@ -19,6 +19,52 @@ export function isIPPrivate(address: string): boolean {
return addr.range() !== "unicast";
}
+/**
+ * Reject private IP literals and localhost names before dispatch, on the
+ * initial request and every redirect hop. With PROXY_SERVER set, the socket
+ * check below never sees the destination, so this is the only local guard.
+ * Other hostnames are resolved proxy-side and must be filtered by the proxy.
+ */
+export const rejectPrivateIPLiteralTargets: undici.Dispatcher.DispatcherComposeInterceptor =
+ dispatch => (options, handler) => {
+ if (config.ALLOW_LOCAL_WEBHOOKS === true || options.origin === undefined) {
+ return dispatch(options, handler);
+ }
+
+ let privateTarget = false;
+
+ try {
+ const origin =
+ options.origin instanceof URL
+ ? options.origin
+ : new URL(options.origin.toString());
+ const hostname = origin.hostname.replace(/^\[|\]$/g, "");
+ privateTarget =
+ isIPPrivate(hostname) || /(^|\.)localhost\.?$/.test(hostname);
+ } catch {
+ // Let Undici report malformed origins through its normal path.
+ }
+
+ if (privateTarget) {
+ const error = new InsecureConnectionError();
+ const compatibleHandler = handler as typeof handler & {
+ onResponseError?: (controller: unknown, error: Error) => void;
+ };
+
+ if (typeof compatibleHandler.onError === "function") {
+ compatibleHandler.onError(error);
+ } else if (typeof compatibleHandler.onResponseError === "function") {
+ compatibleHandler.onResponseError(null, error);
+ } else {
+ throw error;
+ }
+
+ return true;
+ }
+
+ return dispatch(options, handler);
+ };
+
function createBaseAgent(skipTlsVerification: boolean) {
const baseAgent = config.PROXY_SERVER
? new undici.ProxyAgent({
@@ -39,7 +85,10 @@ function createBaseAgent(skipTlsVerification: boolean) {
});
// Add redirect interceptor for handling redirects
- return baseAgent.compose(interceptors.redirect({ maxRedirections: 5000 }));
+ return baseAgent.compose(
+ rejectPrivateIPLiteralTargets,
+ interceptors.redirect({ maxRedirections: 5000 }),
+ );
}
function attachSecurityCheck(agent: undici.Dispatcher) {