Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(playwright-service): harden against SSRF (#3143)

Gergő Móricza2de2bc5e6ad
diff --git a/apps/api/src/__tests__/snips/v2/scrape.test.ts b/apps/api/src/__tests__/snips/v2/scrape.test.ts
index d3c2c9601..16e537629 100644
--- a/apps/api/src/__tests__/snips/v2/scrape.test.ts
+++ b/apps/api/src/__tests__/snips/v2/scrape.test.ts
@@ -27,8 +27,10 @@ import {
} from "./lib";
import request from "./lib";
import crypto from "crypto";
+import { z } from "zod";
const CHANGE_TRACKING_TEST_URL = `${TEST_SUITE_WEBSITE}?testId=${crypto.randomUUID()}`;
+const stringbool = z.stringbool().catch(false);
let identity: Identity;
@@ -52,6 +54,20 @@ beforeAll(async () => {
describe("Scrape tests", () => {
const base = TEST_SUITE_WEBSITE;
+ const playwrightAllowsLocalTargets = stringbool.parse(
+ process.env.ALLOW_LOCAL_WEBHOOKS,
+ );
+ const createSelfHostedLocalUrl = () => {
+ const target = new URL(TEST_SUITE_WEBSITE);
+ target.searchParams.set("testId", crypto.randomUUID());
+ return target.toString();
+ };
+
+ const createDnsResolvedLocalUrl = () => {
+ const target = new URL(createSelfHostedLocalUrl());
+ target.hostname = "localtest.me";
+ return target.toString();
+ };
concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
"works",
@@ -279,6 +295,50 @@ describe("Scrape tests", () => {
scrapeTimeout,
);
+ concurrentIf(
+ TEST_SELF_HOST &&
+ HAS_PLAYWRIGHT &&
+ ALLOW_TEST_SUITE_WEBSITE &&
+ playwrightAllowsLocalTargets,
+ )(
+ "playwright allows local-network targets when ALLOW_LOCAL_WEBHOOKS is enabled",
+ async () => {
+ const response = await scrape(
+ {
+ url: createSelfHostedLocalUrl(),
+ waitFor: 100,
+ },
+ identity,
+ );
+
+ expect(response.markdown).toContain("Firecrawl");
+ },
+ scrapeTimeout,
+ );
+
+ concurrentIf(
+ TEST_SELF_HOST && HAS_PLAYWRIGHT && !playwrightAllowsLocalTargets,
+ )(
+ "playwright blocks local-network targets resolved via DNS",
+ async () => {
+ const raw = await scrapeRaw(
+ {
+ url: createDnsResolvedLocalUrl(),
+ waitFor: 100,
+ },
+ identity,
+ );
+
+ expect(raw.statusCode).toBe(200);
+ expect(raw.body.success).toBe(true);
+ expect(raw.body.data?.metadata?.statusCode).toBe(403);
+ expect(raw.body.data?.metadata?.error).toContain(
+ "Blocked insecure target URL",
+ );
+ },
+ scrapeTimeout,
+ );
+
concurrentIf(TEST_PRODUCTION || (HAS_PLAYWRIGHT && ALLOW_TEST_SUITE_WEBSITE))(
"waitFor works",
async () => {
diff --git a/apps/playwright-service-ts/api.ts b/apps/playwright-service-ts/api.ts
index 4e0534a6b..7d5e8f23e 100644
--- a/apps/playwright-service-ts/api.ts
+++ b/apps/playwright-service-ts/api.ts
@@ -3,6 +3,8 @@ import { chromium, Browser, BrowserContext, Route, Request as PlaywrightRequest,
import dotenv from 'dotenv';
import UserAgent from 'user-agents';
import { getError } from './helpers/get_error';
+import { lookup } from 'dns/promises';
+import IPAddr from 'ipaddr.js';
dotenv.config();
@@ -13,10 +15,106 @@ app.use(express.json());
const BLOCK_MEDIA = (process.env.BLOCK_MEDIA || 'False').toUpperCase() === 'TRUE';
const MAX_CONCURRENT_PAGES = Math.max(1, Number.parseInt(process.env.MAX_CONCURRENT_PAGES ?? '10', 10) || 10);
+const ALLOW_LOCAL_WEBHOOKS = (process.env.ALLOW_LOCAL_WEBHOOKS || 'False').toUpperCase() === 'TRUE';
+const DNS_CACHE_TTL_MS = 30_000;
const PROXY_SERVER = process.env.PROXY_SERVER || null;
const PROXY_USERNAME = process.env.PROXY_USERNAME || null;
const PROXY_PASSWORD = process.env.PROXY_PASSWORD || null;
+const dnsLookupCache = new Map<string, { addresses: string[]; expiresAt: number }>();
+
+class InsecureConnectionError extends Error {
+ constructor(public readonly blockedUrl: string, reason: string) {
+ super(`Blocked insecure target URL "${blockedUrl}": ${reason}`);
+ this.name = 'InsecureConnectionError';
+ }
+}
+
+const normalizeHostname = (hostname: string): string => hostname.toLowerCase().replace(/\.$/, '');
+
+const isHttpProtocol = (protocol: string): boolean => protocol === 'http:' || protocol === 'https:';
+
+const isIPPrivate = (address: string): boolean => {
+ if (!IPAddr.isValid(address)) return false;
+ const parsedAddress = IPAddr.parse(address);
+ return parsedAddress.range() !== 'unicast';
+};
+
+const isLocalHostname = (hostname: string): boolean =>
+ hostname === 'localhost' || hostname.endsWith('.localhost');
+
+const lookupWithCache = async (hostname: string): Promise<string[]> => {
+ const cached = dnsLookupCache.get(hostname);
+ if (cached && cached.expiresAt > Date.now()) {
+ return cached.addresses;
+ }
+
+ const resolvedAddresses = await lookup(hostname, { all: true, verbatim: true });
+ const uniqueAddresses = [...new Set(resolvedAddresses.map(x => x.address))];
+ dnsLookupCache.set(hostname, {
+ addresses: uniqueAddresses,
+ expiresAt: Date.now() + DNS_CACHE_TTL_MS,
+ });
+ return uniqueAddresses;
+};
+
+const assertSafeTargetUrl = async (urlString: string): Promise<void> => {
+ let parsedUrl: URL;
+ try {
+ parsedUrl = new URL(urlString);
+ } catch {
+ throw new InsecureConnectionError(urlString, 'URL is invalid');
+ }
+
+ if (!isHttpProtocol(parsedUrl.protocol)) {
+ throw new InsecureConnectionError(urlString, `unsupported protocol "${parsedUrl.protocol}"`);
+ }
+
+ if (ALLOW_LOCAL_WEBHOOKS) {
+ return;
+ }
+
+ const hostname = normalizeHostname(parsedUrl.hostname);
+ if (!hostname) {
+ throw new InsecureConnectionError(urlString, 'hostname is missing');
+ }
+
+ if (isLocalHostname(hostname)) {
+ throw new InsecureConnectionError(urlString, 'localhost targets are not allowed');
+ }
+
+ if (IPAddr.isValid(hostname)) {
+ if (isIPPrivate(hostname)) {
+ throw new InsecureConnectionError(urlString, `private IP "${hostname}" is not allowed`);
+ }
+ return;
+ }
+
+ let resolvedAddresses: string[];
+ try {
+ resolvedAddresses = await lookupWithCache(hostname);
+ } catch {
+ throw new InsecureConnectionError(
+ urlString,
+ `DNS lookup failed for "${hostname}", cannot verify target is safe`,
+ );
+ }
+
+ if (resolvedAddresses.length === 0) {
+ throw new InsecureConnectionError(
+ urlString,
+ `hostname "${hostname}" did not resolve to any IP address`,
+ );
+ }
+
+ if (resolvedAddresses.some(address => isIPPrivate(address))) {
+ throw new InsecureConnectionError(urlString, `hostname "${hostname}" resolves to a private IP`);
+ }
+};
+
+type ContextSecurityState = {
+ blockedNavigationRequestUrl: string | null;
+};
class Semaphore {
private permits: number;
private queue: (() => void)[] = [];
@@ -99,14 +197,18 @@ const initializeBrowser = async () => {
});
};
-const createContext = async (skipTlsVerification: boolean = false) => {
+const createContext = async (skipTlsVerification: boolean = false): Promise<{ context: BrowserContext; securityState: ContextSecurityState }> => {
const userAgent = new UserAgent().toString();
const viewport = { width: 1280, height: 800 };
+ const securityState: ContextSecurityState = {
+ blockedNavigationRequestUrl: null,
+ };
const contextOptions: any = {
userAgent,
viewport,
ignoreHTTPSErrors: skipTlsVerification,
+ serviceWorkers: 'block',
};
if (PROXY_SERVER && PROXY_USERNAME && PROXY_PASSWORD) {
@@ -130,9 +232,23 @@ const createContext = async (skipTlsVerification: boolean = false) => {
}
// Intercept all requests to avoid loading ads
- await newContext.route('**/*', (route: Route, request: PlaywrightRequest) => {
- const requestUrl = new URL(request.url());
- const hostname = requestUrl.hostname;
+ await newContext.route('**/*', async (route: Route, request: PlaywrightRequest) => {
+ const requestUrlString = request.url();
+ try {
+ await assertSafeTargetUrl(requestUrlString);
+ } catch (error) {
+ if (error instanceof InsecureConnectionError) {
+ if (request.isNavigationRequest()) {
+ securityState.blockedNavigationRequestUrl = requestUrlString;
+ }
+ console.warn(`Blocked request: ${requestUrlString}`);
+ return route.abort('blockedbyclient');
+ }
+ throw error;
+ }
+
+ const requestUrl = new URL(requestUrlString);
+ const hostname = normalizeHostname(requestUrl.hostname);
if (AD_SERVING_DOMAINS.some(domain => hostname.includes(domain))) {
console.log(hostname);
@@ -141,7 +257,7 @@ const createContext = async (skipTlsVerification: boolean = false) => {
return route.continue();
});
- return newContext;
+ return { context: newContext, securityState };
};
const shutdownBrowser = async () => {
@@ -159,9 +275,28 @@ const isValidUrl = (urlString: string): boolean => {
}
};
-const scrapePage = async (page: Page, url: string, waitUntil: 'load' | 'networkidle', waitAfterLoad: number, timeout: number, checkSelector: string | undefined) => {
+const scrapePage = async (
+ page: Page,
+ url: string,
+ waitUntil: 'load' | 'networkidle',
+ waitAfterLoad: number,
+ timeout: number,
+ checkSelector: string | undefined,
+ securityState: ContextSecurityState,
+) => {
console.log(`Navigating to ${url} with waitUntil: ${waitUntil} and timeout: ${timeout}ms`);
- const response = await page.goto(url, { waitUntil, timeout });
+ let response;
+ try {
+ response = await page.goto(url, { waitUntil, timeout });
+ } catch (error) {
+ if (securityState.blockedNavigationRequestUrl) {
+ throw new InsecureConnectionError(
+ securityState.blockedNavigationRequestUrl,
+ 'navigation to private/internal resource is not allowed',
+ );
+ }
+ throw error;
+ }
if (waitAfterLoad > 0) {
await page.waitForTimeout(waitAfterLoad);
@@ -199,7 +334,7 @@ app.get('/health', async (req: Request, res: Response) => {
await initializeBrowser();
}
- const testContext = await createContext();
+ const { context: testContext } = await createContext();
const testPage = await testContext.newPage();
await testPage.close();
await testContext.close();
@@ -238,6 +373,19 @@ app.post('/scrape', async (req: Request, res: Response) => {
return res.status(400).json({ error: 'Invalid URL' });
}
+ try {
+ await assertSafeTargetUrl(url);
+ } catch (error) {
+ if (error instanceof InsecureConnectionError) {
+ return res.json({
+ content: '',
+ pageStatusCode: 403,
+ pageError: error.message,
+ });
+ }
+ throw error;
+ }
+
if (!PROXY_SERVER) {
console.warn('⚠️ WARNING: No proxy server provided. Your IP address may be blocked.');
}
@@ -249,17 +397,28 @@ app.post('/scrape', async (req: Request, res: Response) => {
await pageSemaphore.acquire();
let requestContext: BrowserContext | null = null;
+ let securityState: ContextSecurityState | null = null;
let page: Page | null = null;
try {
- requestContext = await createContext(skip_tls_verification);
+ const contextBundle = await createContext(skip_tls_verification);
+ requestContext = contextBundle.context;
+ securityState = contextBundle.securityState;
page = await requestContext.newPage();
if (headers) {
await page.setExtraHTTPHeaders(headers);
}
- const result = await scrapePage(page, url, 'load', wait_after_load, timeout, check_selector);
+ const result = await scrapePage(
+ page,
+ url,
+ 'load',
+ wait_after_load,
+ timeout,
+ check_selector,
+ securityState,
+ );
const pageError = result.status !== 200 ? getError(result.status) : undefined;
if (!pageError) {
@@ -276,6 +435,13 @@ app.post('/scrape', async (req: Request, res: Response) => {
});
} catch (error) {
+ if (error instanceof InsecureConnectionError) {
+ return res.json({
+ content: '',
+ pageStatusCode: 403,
+ pageError: error.message,
+ });
+ }
console.error('Scrape error:', error);
res.status(500).json({ error: 'An error occurred while fetching the page.' });
} finally {
diff --git a/apps/playwright-service-ts/package.json b/apps/playwright-service-ts/package.json
index ebf1df69d..ead74c048 100644
--- a/apps/playwright-service-ts/package.json
+++ b/apps/playwright-service-ts/package.json
@@ -14,6 +14,7 @@
"dependencies": {
"dotenv": "^16.4.5",
"express": "^5.2.1",
+ "ipaddr.js": "^2.3.0",
"playwright": "^1.58.1",
"user-agents": "^1.1.669"
},
diff --git a/apps/playwright-service-ts/pnpm-lock.yaml b/apps/playwright-service-ts/pnpm-lock.yaml
index fc834c420..fde4d0c91 100644
--- a/apps/playwright-service-ts/pnpm-lock.yaml
+++ b/apps/playwright-service-ts/pnpm-lock.yaml
@@ -14,6 +14,9 @@ importers:
express:
specifier: ^5.2.1
version: 5.2.1
+ ipaddr.js:
+ specifier: ^2.3.0
+ version: 2.3.0
playwright:
specifier: ^1.58.1
version: 1.58.1
@@ -386,6 +389,10 @@ packages:
resolution: {integrity: sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==}
engines: {node: '>= 0.10'}
+ resolution: {integrity: sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg==}
+ engines: {node: '>= 10'}
+
resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==}
@@ -872,6 +879,8 @@ snapshots:
+
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 0dfe14f8f..d925f80f7 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -67,6 +67,7 @@ services:
PROXY_SERVER: ${PROXY_SERVER}
PROXY_USERNAME: ${PROXY_USERNAME}
PROXY_PASSWORD: ${PROXY_PASSWORD}
+ ALLOW_LOCAL_WEBHOOKS: ${ALLOW_LOCAL_WEBHOOKS}
BLOCK_MEDIA: ${BLOCK_MEDIA}
# Configure maximum concurrent pages for Playwright browser instances
MAX_CONCURRENT_PAGES: ${CRAWL_CONCURRENT_REQUESTS:-10}
diff --git a/examples/kubernetes/cluster-install/playwright-service.yaml b/examples/kubernetes/cluster-install/playwright-service.yaml
index b40cfa835..5298a4584 100644
--- a/examples/kubernetes/cluster-install/playwright-service.yaml
+++ b/examples/kubernetes/cluster-install/playwright-service.yaml
@@ -4,6 +4,7 @@ metadata:
name: playwright-service-config
data:
PORT: "3000"
+ ALLOW_LOCAL_WEBHOOKS: "false"
---
apiVersion: apps/v1
kind: Deployment
diff --git a/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml b/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml
index f3be19085..bab3e047a 100644
--- a/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml
+++ b/examples/kubernetes/firecrawl-helm/templates/playwright-configmap.yaml
@@ -4,5 +4,6 @@ metadata:
name: {{ include "firecrawl.fullname" . }}-playwright-config
data:
PORT: {{ default (printf "%v" .Values.service.playwright.port) .Values.playwrightConfig.PORT | quote }}
+ ALLOW_LOCAL_WEBHOOKS: {{ .Values.playwrightConfig.ALLOW_LOCAL_WEBHOOKS | quote }}
BLOCK_MEDIA: {{ .Values.playwrightConfig.BLOCK_MEDIA | quote }}
MAX_CONCURRENT_PAGES: {{ .Values.playwrightConfig.MAX_CONCURRENT_PAGES | quote }}
diff --git a/examples/kubernetes/firecrawl-helm/values.yaml b/examples/kubernetes/firecrawl-helm/values.yaml
index afa9868fb..b809ee188 100644
--- a/examples/kubernetes/firecrawl-helm/values.yaml
+++ b/examples/kubernetes/firecrawl-helm/values.yaml
@@ -160,6 +160,7 @@ config:
playwrightConfig:
PORT: "3000"
+ ALLOW_LOCAL_WEBHOOKS: ""
BLOCK_MEDIA: ""
MAX_CONCURRENT_PAGES: "10"