Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(playwright-service): pin SSRF proxy connections to the checked DNS answer (#5261)
posthog[bot]5e74ff37b41a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex e68b58293..582be7321 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -180,6 +180,11 @@ jobs:run: pnpm install --frozen-lockfileworking-directory: ./apps/playwright-service-ts+ - name: Run Playwright service unit tests+ if: matrix.engine == 'playwright' && matrix.proxy == 'no-proxy'+ run: pnpm test+ working-directory: ./apps/playwright-service-ts+- name: Install Playwright system depsif: matrix.engine == 'playwright'run: pnpm exec playwright install-deps chromiumdiff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.tsindex 1d0fa7f12..5821a9464 100644--- a/apps/playwright-service-ts/api.ts+++ b/apps/playwright-service-ts/api.ts@@ -10,9 +10,8 @@ import {import dotenv from 'dotenv';import UserAgent from 'user-agents';import { getError } from './helpers/get_error';-import { lookup } from 'dns/promises';-import IPAddr from 'ipaddr.js';-import { Server, RequestError } from 'proxy-chain';+import { Server } from 'proxy-chain';+import { isInternalHost, ssrfPrepareRequest } from './helpers/ssrf';dotenv.config();@@ -44,26 +43,6 @@ class InsecureConnectionError extends Error {}}-const isInternalHost = async (hostname: string): Promise<boolean> => {- const host = hostname.toLowerCase().replace(/\.$/, '');- if (!host) return true;-- let addresses: string[];- if (IPAddr.isValid(host)) {- addresses = [host];- } else {- try {- addresses = (await lookup(host, { all: true })).map((a) => a.address);- } catch {- return true;- }- }- return (- addresses.length === 0 ||- addresses.some((a) => IPAddr.parse(a).range() !== 'unicast')- );-};-const assertSafeTargetUrl = async (urlString: string): Promise<void> => {let parsedUrl: URL;try {@@ -100,15 +79,10 @@ const startSSRFProxy = async (): Promise<number> => {const server = new Server({port: 0,host: '127.0.0.1',- prepareRequestFunction: async ({ hostname }) => {- if (!ALLOW_LOCAL_WEBHOOKS && (await isInternalHost(hostname))) {- throw new RequestError(- 'Blocked: target resolves to a private/internal address',- 403,- );- }- return { upstreamProxyUrl: buildUpstreamProxyUrl() };- },+ prepareRequestFunction: ssrfPrepareRequest({+ allowLocal: ALLOW_LOCAL_WEBHOOKS,+ upstreamProxyUrl: buildUpstreamProxyUrl(),+ }),});await server.listen();return server.port;diff --git a/apps/playwright-service-ts/helpers/ssrf.spec.ts b/apps/playwright-service-ts/helpers/ssrf.spec.tsnew file mode 100644index 000000000..05031254e--- /dev/null+++ b/apps/playwright-service-ts/helpers/ssrf.spec.ts@@ -0,0 +1,187 @@+import { test } from 'node:test';+import assert from 'node:assert/strict';+import http from 'node:http';+import net, { AddressInfo } from 'node:net';+import { LookupAddress } from 'node:dns';+import { Server } from 'proxy-chain';+import {+ pinnedLookup,+ resolvePublicAddresses,+ ssrfPrepareRequest,+} from './ssrf';++const PUBLIC_IP = '8.8.8.8';++const listen = async (server: net.Server): Promise<number> => {+ await new Promise<void>((resolve) => server.listen(0, '127.0.0.1', resolve));+ return (server.address() as AddressInfo).port;+};++const startTarget = async () => {+ let hits = 0;+ const server = http.createServer((_req, res) => {+ hits++;+ res.end('internal');+ });+ const port = await listen(server);+ return { port, hits: () => hits, close: () => server.close() };+};++const startProxy = async (resolve: (h: string) => Promise<LookupAddress[]>) => {+ const server = new Server({+ port: 0,+ host: '127.0.0.1',+ prepareRequestFunction: ssrfPrepareRequest({ allowLocal: false, resolve }),+ });+ await server.listen();+ return { port: server.port, close: () => server.close(true) };+};++const requestViaProxy = (+ proxyPort: number,+ url: string,+): Promise<number | 'error'> =>+ new Promise((resolve) => {+ const req = http.request({+ host: '127.0.0.1',+ port: proxyPort,+ path: url,+ headers: { host: new URL(url).host },+ timeout: 1000,+ });+ req.on('response', (res) => {+ res.resume();+ resolve(res.statusCode ?? 0);+ });+ req.on('timeout', () => req.destroy());+ req.on('error', () => resolve('error'));+ req.end();+ });++const connectViaProxy = (proxyPort: number, target: string): Promise<number> =>+ new Promise((resolve, reject) => {+ const req = http.request({+ host: '127.0.0.1',+ port: proxyPort,+ method: 'CONNECT',+ path: target,+ timeout: 1000,+ });+ req.on('connect', (res, socket) => {+ socket.destroy();+ resolve(res.statusCode ?? 0);+ });+ req.on('timeout', () => req.destroy(new Error('Proxy CONNECT timed out')));+ req.on('error', reject);+ req.end();+ });++test('resolvePublicAddresses rejects non-unicast addresses', async () => {+ for (const ip of [+ '127.0.0.1',+ '10.0.0.1',+ '169.254.169.254',+ '::1',+ '::ffff:127.0.0.1',+ '::ffff:169.254.169.254',+ ]) {+ assert.equal(await resolvePublicAddresses(ip), null, ip);+ }+ assert.deepEqual(await resolvePublicAddresses('8.8.8.8'), [+ { address: '8.8.8.8', family: 4 },+ ]);+});++test('resolvePublicAddresses rejects a host if any address is internal', async () => {+ const mixed = async () => [+ { address: PUBLIC_IP, family: 4 },+ { address: '10.0.0.1', family: 4 },+ ];+ assert.equal(await resolvePublicAddresses('mixed.test', mixed), null);+ assert.equal(+ await resolvePublicAddresses('empty.test', async () => []),+ null,+ );+ assert.equal(+ await resolvePublicAddresses('fail.test', async () => {+ throw new Error('ENOTFOUND');+ }),+ null,+ );+});++test('pinnedLookup makes the socket connect to the pinned address without DNS', async () => {+ const server = net.createServer((s) => s.end());+ const port = await listen(server);+ try {+ const remote = await new Promise<string | undefined>((resolve, reject) => {+ const socket = net.connect({+ host: 'never-resolved.invalid',+ port,+ lookup: pinnedLookup([{ address: '127.0.0.1', family: 4 }]),+ });+ socket.on('connect', () => {+ resolve(socket.remoteAddress);+ socket.destroy();+ });+ socket.on('error', reject);+ });+ assert.equal(remote, '127.0.0.1');+ } finally {+ server.close();+ }+});++test('proxy returns 403 for a host that resolves to a private address', async () => {+ const target = await startTarget();+ const proxy = await startProxy(async () => [+ { address: '127.0.0.1', family: 4 },+ ]);+ try {+ assert.equal(+ await requestViaProxy(proxy.port, `http://private.test:${target.port}/`),+ 403,+ );+ assert.equal(+ await connectViaProxy(proxy.port, `private.test:${target.port}`),+ 403,+ );+ assert.equal(target.hits(), 0);+ } finally {+ await proxy.close();+ target.close();+ }+});++test('proxy does not resolve the host again at connect time (DNS rebinding)', async () => {+ // The check sees a public address. A second lookup of "localhost" through+ // the system resolver would return loopback, as a rebinding record would.+ const target = await startTarget();+ let calls = 0;+ const proxy = await startProxy(async () => {+ calls++;+ return [{ address: PUBLIC_IP, family: 4 }];+ });+ try {+ const status = await requestViaProxy(+ proxy.port,+ `http://localhost:${target.port}/`,+ );+ assert.notEqual(status, 200);+ assert.equal(calls, 1);+ assert.equal(target.hits(), 0);+ } finally {+ await proxy.close();+ target.close();+ }+});++test('with an upstream proxy, no dnsLookup is pinned', async () => {+ const prepare = ssrfPrepareRequest({+ allowLocal: false,+ upstreamProxyUrl: 'http://proxy.internal:8080',+ resolve: async () => [{ address: PUBLIC_IP, family: 4 }],+ });+ const result = await prepare({ hostname: 'example.test' } as never);+ assert.deepEqual(result, { upstreamProxyUrl: 'http://proxy.internal:8080' });+});diff --git a/apps/playwright-service-ts/helpers/ssrf.ts b/apps/playwright-service-ts/helpers/ssrf.tsnew file mode 100644index 000000000..3e72c4c35--- /dev/null+++ b/apps/playwright-service-ts/helpers/ssrf.ts@@ -0,0 +1,108 @@+import dns, { LookupAddress } from 'dns';+import { lookup } from 'dns/promises';+import IPAddr from 'ipaddr.js';+import { PrepareRequestFunction, RequestError } from 'proxy-chain';++type Resolver = (hostname: string) => Promise<LookupAddress[]>;++const defaultResolver: Resolver = (hostname) => lookup(hostname, { all: true });++/**+ * Resolves `hostname` once. Returns the addresses if all of them are public+ * unicast, or null if the host is empty, does not resolve, or has any+ * non-unicast address (private, loopback, link-local, IPv4-mapped IPv6, ...).+ */+export const resolvePublicAddresses = async (+ hostname: string,+ resolve: Resolver = defaultResolver,+): Promise<LookupAddress[] | null> => {+ const host = hostname.toLowerCase().replace(/\.$/, '');+ if (!host) return null;++ let addresses: LookupAddress[];+ if (IPAddr.isValid(host)) {+ const ip = IPAddr.parse(host);+ addresses = [+ { address: ip.toString(), family: ip.kind() === 'ipv6' ? 6 : 4 },+ ];+ } else {+ try {+ addresses = await resolve(host);+ } catch {+ return null;+ }+ }+ if (+ addresses.length === 0 ||+ addresses.some((a) => IPAddr.parse(a.address).range() !== 'unicast')+ ) {+ return null;+ }+ return addresses;+};++export const isInternalHost = async (hostname: string): Promise<boolean> =>+ (await resolvePublicAddresses(hostname)) === null;++/**+ * A `dns.lookup` replacement that answers only with addresses that were+ * already checked, so the socket connects to the address the guard approved+ * and Node never resolves the host a second time.+ */+export const pinnedLookup = (addresses: LookupAddress[]): typeof dns.lookup =>+ ((+ _hostname: string,+ opts: dns.LookupOptions,+ cb: (+ err: NodeJS.ErrnoException | null,+ address?: string | LookupAddress[],+ family?: number,+ ) => void,+ ) => {+ const family = opts.family;+ const matches =+ family === 4 || family === 6+ ? addresses.filter((a) => a.family === family)+ : addresses;++ process.nextTick(() => {+ if (matches.length === 0) {+ const err: NodeJS.ErrnoException = new Error(+ `No pinned address for family ${family}`,+ );+ err.code = 'ENOTFOUND';+ cb(err);+ } else if (opts.all) {+ cb(null, matches);+ } else {+ cb(null, matches[0].address, matches[0].family);+ }+ });+ }) as typeof dns.lookup;++export const ssrfPrepareRequest =+ ({+ allowLocal,+ upstreamProxyUrl,+ resolve,+ }: {+ allowLocal: boolean;+ upstreamProxyUrl?: string;+ resolve?: Resolver;+ }): PrepareRequestFunction =>+ async ({ hostname }) => {+ if (allowLocal) return { upstreamProxyUrl };++ const addresses = await resolvePublicAddresses(hostname, resolve);+ if (!addresses) {+ throw new RequestError(+ 'Blocked: target resolves to a private/internal address',+ 403,+ );+ }+ // With an upstream proxy, proxy-chain uses dnsLookup for the proxy host,+ // and the upstream proxy resolves the target itself. That proxy must+ // enforce its own egress policy; this check is only a pre-check there.+ if (upstreamProxyUrl) return { upstreamProxyUrl };+ return { dnsLookup: pinnedLookup(addresses) };+ };diff --git a/apps/playwright-service-ts/package.json b/apps/playwright-service-ts/package.jsonindex e905ac004..9a14e86fd 100644--- a/apps/playwright-service-ts/package.json+++ b/apps/playwright-service-ts/package.json@@ -6,7 +6,8 @@"scripts": {"start": "node dist/api.js","build": "tsc",- "dev": "tsx api.ts"+ "dev": "tsx api.ts",+ "test": "node --import tsx --test --test-force-exit helpers/*.spec.ts"},"keywords": [],"author": "Jeff Pereira",