Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): bound the cost of compiling includePaths/excludePaths patterns (#4533)

Gaurav Chadhaba5442250215
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index bd823b7ed..40fc457fa 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -305,15 +305,15 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
Url::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;
let scope = crawl_scope(&initial_url);
- let excludes_regex: Vec<Regex> = data
+ let excludes_regex: Vec<PathRegex> = data
.excludes
.iter()
- .filter_map(|e| Regex::new(e).ok())
+ .filter_map(|e| compile_path_regex(e).ok())
.collect();
- let includes_regex: Vec<Regex> = data
+ let includes_regex: Vec<PathRegex> = data
.includes
.iter()
- .filter_map(|i| Regex::new(i).ok())
+ .filter_map(|i| compile_path_regex(i).ok())
.collect();
let robot = build_robot(
@@ -374,12 +374,20 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
path
};
- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(match_target)) {
+ if !excludes_regex.is_empty()
+ && excludes_regex
+ .iter()
+ .any(|r| r.is_match(match_target.as_bytes()))
+ {
denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());
continue;
}
- if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {
+ if !includes_regex.is_empty()
+ && !includes_regex
+ .iter()
+ .any(|r| r.is_match(match_target.as_bytes()))
+ {
denial_reasons.insert(link, INCLUDE_PATTERN.to_string());
continue;
}
@@ -399,7 +407,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
continue;
}
- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {
+ if !excludes_regex.is_empty()
+ && excludes_regex
+ .iter()
+ .any(|r| r.is_match(url_str.as_bytes()))
+ {
denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());
continue;
}
@@ -422,7 +434,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult
} else {
path
};
- if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {
+ if !includes_regex.is_empty()
+ && !includes_regex
+ .iter()
+ .any(|r| r.is_match(match_target.as_bytes()))
+ {
denial_reasons.insert(link, INCLUDE_PATTERN.to_string());
continue;
}
@@ -455,6 +471,47 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))
}
+/// Compiled form of a client-supplied includePaths/excludePaths pattern.
+///
+/// Path patterns are compiled with Unicode mode disabled, over bytes. Every
+/// haystack they are matched against is a serialized `url::Url` path or href,
+/// which is always percent-encoded ASCII, so ASCII and Unicode mode produce the
+/// same matches. Unicode mode is what makes compilation expensive: a Unicode
+/// `\w` is hundreds of codepoint ranges, so `\w{50}` alone compiles to more than
+/// two megabytes of NFA, and `.{2000}` or `[\w-]{1,100}` several times over hit
+/// the crate's 10 MiB default. In ASCII mode the same patterns compile in well
+/// under a millisecond, which is what lets the size limit below be tight.
+type PathRegex = regex::bytes::Regex;
+
+/// Approximate upper bound, in bytes, on the compiled NFA of a single path
+/// pattern. Patterns are untrusted, and the `regex` crate's compile time and
+/// heap usage scale with the *compiled* size rather than the pattern's length:
+/// `a{5}{5}{5}{5}{5}{5}` is 19 characters but expands to `a{15625}`. The crate's
+/// default of 10 MiB lets a short pattern cost tens of milliseconds of CPU per
+/// compile; this limit keeps it well under one millisecond while still admitting
+/// path filters far larger than any realistic one.
+/// https://docs.rs/regex/latest/regex/#untrusted-patterns
+const PATH_REGEX_SIZE_LIMIT: usize = 256 * 1024;
+
+/// Cache capacity of the lazy DFA used when matching a path pattern. Bounded so
+/// a large pattern cannot claim the crate's 2 MiB default per compiled regex.
+const PATH_REGEX_DFA_SIZE_LIMIT: usize = 256 * 1024;
+
+/// Maximum nesting depth of a path pattern's syntax tree (crate default: 250).
+const PATH_REGEX_NEST_LIMIT: u32 = 50;
+
+/// Compile an includePaths/excludePaths pattern with resource limits suitable
+/// for untrusted input. Every place that compiles a client-supplied path pattern
+/// must go through this so validation and filtering agree on what is accepted.
+fn compile_path_regex(pattern: &str) -> std::result::Result<PathRegex, regex::Error> {
+ regex::bytes::RegexBuilder::new(pattern)
+ .unicode(false)
+ .size_limit(PATH_REGEX_SIZE_LIMIT)
+ .dfa_size_limit(PATH_REGEX_DFA_SIZE_LIMIT)
+ .nest_limit(PATH_REGEX_NEST_LIMIT)
+ .build()
+}
+
/// Validate regex patterns against the engine's regex flavor (the Rust `regex`
/// crate), which is what link filtering compiles them with. Returns an entry for
/// each pattern that fails to compile so callers can reject unsupported syntax
@@ -463,7 +520,7 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
pub fn validate_regexes(patterns: Vec<String>) -> Vec<RegexValidationError> {
patterns
.into_iter()
- .filter_map(|pattern| match Regex::new(&pattern) {
+ .filter_map(|pattern| match compile_path_regex(&pattern) {
Ok(_) => None,
Err(e) => Some(RegexValidationError {
pattern,
@@ -532,10 +589,10 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
});
}
- let excludes_regex: Vec<Regex> = data
+ let excludes_regex: Vec<PathRegex> = data
.excludes
.iter()
- .filter_map(|e| Regex::new(e).ok())
+ .filter_map(|e| compile_path_regex(e).ok())
.collect();
let robot = build_robot(
@@ -554,7 +611,7 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
});
}
- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path)) {
+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path.as_bytes())) {
return Ok(FilterUrlResult {
allowed: false,
url: None,
@@ -587,7 +644,11 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
});
}
- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {
+ if !excludes_regex.is_empty()
+ && excludes_regex
+ .iter()
+ .any(|r| r.is_match(url_str.as_bytes()))
+ {
return Ok(FilterUrlResult {
allowed: false,
url: None,
@@ -613,7 +674,9 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri
// it), matched via the same PSL check used for allowSubdomains.
if data.allow_external_content_links
&& !is_external_main_page(url_str)
- && (is_internal_link(&context_url, &base_url) || is_subdomain(&url, &context_url) || is_internal_link(&url, &context_url))
+ && (is_internal_link(&context_url, &base_url)
+ || is_subdomain(&url, &context_url)
+ || is_internal_link(&url, &context_url))
{
return Ok(FilterUrlResult {
allowed: true,
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
index 765ea4490..47fda3127 100644
--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
@@ -540,6 +540,24 @@ describe("V1 Types Validation", () => {
/look-around, including look-ahead and look-behind, is not supported/,
);
});
+
+ it("should reject patterns whose compiled form exceeds the size limit", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],
+ }),
+ ).toThrow(/exceeds size limit/);
+ });
+
+ it("should reject more than the maximum number of path patterns", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ includePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),
+ }),
+ ).toThrow(/at most 100 patterns/);
+ });
});
describe("mapRequestSchema", () => {
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 367092fef..8d9f813ae 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -191,6 +191,47 @@ describe("Crawl tests", () => {
10 * scrapeTimeout,
);
+ it.concurrent(
+ "rejects path patterns that are too expensive to compile",
+ async () => {
+ // 19 characters that expand to a{15625}. Compiling patterns like this
+ // with the engine's default limits is a CPU denial-of-service vector.
+ const res = await crawlStart(
+ {
+ url: "https://firecrawl.dev",
+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],
+ },
+ identity,
+ );
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toMatch(/exceeds size limit/);
+ },
+ scrapeTimeout,
+ );
+
+ it.concurrent(
+ "rejects more path patterns than the engine will compile",
+ async () => {
+ const res = await crawlStart(
+ {
+ url: "https://firecrawl.dev",
+ excludePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),
+ },
+ identity,
+ );
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ // Schema-level (non-custom) issues are reported in details, not error.
+ expect(
+ res.body.details.map((issue: { message: string }) => issue.message),
+ ).toContainEqual(expect.stringMatching(/at most 100 patterns/));
+ },
+ scrapeTimeout,
+ );
+
// TODO: port to new dynamic url system
// concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
// "filters URLs properly when using regexOnFullURL",
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 40bf6c918..60ebca9c6 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -1,4 +1,8 @@
import { z } from "zod";
+import {
+ MAX_PATH_PATTERNS,
+ MAX_PATH_PATTERN_LENGTH,
+} from "../../../lib/crawl-regex";
import {
scrapeRequestSchema,
parseRequestSchema,
@@ -970,6 +974,105 @@ describe("V2 Types Validation", () => {
expect(message.match(/not supported/g)).toHaveLength(1);
expect(message).toMatch(/Rewrite the pattern/);
});
+
+ it("should accept counted repetitions of word classes that real path filters use", () => {
+ // Unicode \w is hundreds of ranges, so these used to exceed the engine's
+ // compiled-size limit and were silently dropped. Path haystacks are
+ // percent-encoded ASCII, so they are compiled in ASCII mode and are cheap.
+ const result = crawlRequestSchema.parse({
+ url: "https://example.com",
+ includePaths: [
+ "^/[\\w-]{1,100}/[\\w-]{1,100}/[\\w-]{1,100}/?$",
+ "\\w{300}",
+ ],
+ });
+
+ expect(result.includePaths).toHaveLength(2);
+ });
+
+ it("should reject patterns whose compiled form exceeds the size limit", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message).toMatch(/exceeds size limit/);
+ expect(message).toMatch(/stacked counted repetitions/);
+ });
+
+ it("should reject Unicode-only constructs with a hint", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/\\p{Greek}+"],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message).toMatch(/Unicode not allowed/);
+ expect(message).toMatch(/percent-encoded ASCII/);
+ });
+
+ it("should reject more than the maximum number of path patterns", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: Array.from(
+ { length: MAX_PATH_PATTERNS + 1 },
+ (_, i) => `^/p${i}`,
+ ),
+ }),
+ ).toThrow(/at most 100 patterns/);
+ });
+
+ it("should not compile patterns once the count cap is exceeded", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: [
+ ...Array.from({ length: MAX_PATH_PATTERNS }, (_, i) => `^/p${i}`),
+ "[abc",
+ ],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message).toMatch(/at most 100 patterns/);
+ expect(message).not.toMatch(/unclosed character class/);
+ });
+
+ it("should not derive hints from user pattern text", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["[exceeds size limit"],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message).toMatch(/unclosed character class/);
+ expect(message).not.toMatch(/stacked counted repetitions/);
+ });
+
+ it("should reject path patterns longer than the maximum length", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ includePaths: ["^/" + "a".repeat(MAX_PATH_PATTERN_LENGTH)],
+ }),
+ ).toThrow(/at most 2000 characters/);
+ });
});
describe("mapRequestSchema", () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 1b1b5380c..291a02729 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -4,7 +4,7 @@ import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
-import { addPathRegexIssues } from "../../lib/crawl-regex";
+import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";
import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";
import {
ExtractorOptions,
@@ -873,8 +873,8 @@ export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;
export type BatchScrapeRequestInput = z.input<typeof batchScrapeRequestSchema>;
const crawlerOptions = z.strictObject({
- includePaths: z.string().array().prefault([]),
- excludePaths: z.string().array().prefault([]),
+ includePaths: pathPatternsSchema.prefault([]),
+ excludePaths: pathPatternsSchema.prefault([]),
maxDepth: z.number().prefault(10), // default?
maxDiscoveryDepth: z.number().optional(),
limit: z.number().prefault(10000), // default?
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 7c86c1ba1..3f4c717e0 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -5,7 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
import { includesFormat } from "../../lib/format-utils";
-import { addPathRegexIssues } from "../../lib/crawl-regex";
+import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";
import {
ExtractorOptions,
PageOptions,
@@ -1268,8 +1268,8 @@ export type BatchScrapeRequestInput = Omit<
};
export const crawlerOptions = z.strictObject({
- includePaths: z.string().array().prefault([]),
- excludePaths: z.string().array().prefault([]),
+ includePaths: pathPatternsSchema.prefault([]),
+ excludePaths: pathPatternsSchema.prefault([]),
maxDiscoveryDepth: z.number().optional(),
limit: z.number().prefault(10000), // default?
crawlEntireDomain: z.boolean().optional(),
diff --git a/apps/api/src/lib/crawl-regex.ts b/apps/api/src/lib/crawl-regex.ts
index 0efda29a9..c811bfa5b 100644
--- a/apps/api/src/lib/crawl-regex.ts
+++ b/apps/api/src/lib/crawl-regex.ts
@@ -1,6 +1,26 @@
import { validateRegexes } from "@mendable/firecrawl-rs";
import { z } from "zod";
+// Every pattern is compiled by the engine at request time (validation) and again
+// when links are filtered, so the amount of work a single request can demand is
+// count x per-pattern cost. The engine bounds per-pattern cost (see
+// compile_path_regex in native/src/crawler.rs); these bound the count and the
+// pattern length, which is what parsing cost scales with.
+export const MAX_PATH_PATTERNS = 100;
+export const MAX_PATH_PATTERN_LENGTH = 2000;
+
+export const pathPatternsSchema = z
+ .string()
+ .max(
+ MAX_PATH_PATTERN_LENGTH,
+ `Each includePaths/excludePaths pattern must be at most ${MAX_PATH_PATTERN_LENGTH} characters.`,
+ )
+ .array()
+ .max(
+ MAX_PATH_PATTERNS,
+ `includePaths and excludePaths each accept at most ${MAX_PATH_PATTERNS} patterns.`,
+ );
+
// Link filtering compiles includePaths/excludePaths with the Rust `regex` crate
// (RE2-style: no look-around or backreferences). Historically an unsupported
// pattern compiled fine in most clients' regex flavor but was silently dropped
@@ -12,18 +32,24 @@ export function addPathRegexIssues(
ctx: z.RefinementCtx,
): void {
if (!patterns || patterns.length === 0) return;
+ // Zod still runs this refinement when pathPatternsSchema has already reported
+ // a count or length violation, so re-check the caps here: they are what bound
+ // the total compile work a request can demand, and an over-cap request has
+ // already been rejected.
+ if (
+ patterns.length > MAX_PATH_PATTERNS ||
+ patterns.some(p => p.length > MAX_PATH_PATTERN_LENGTH)
+ ) {
+ return;
+ }
for (const { pattern, error } of validateRegexes(patterns)) {
- // The engine's own error already names look-around/backreferences as
- // unsupported, so only add the actionable rewrite hint in that case.
- const hint = /look-around|look-ahead|look-behind|backreference/i.test(error)
- ? " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead."
- : "";
+ const summary = summarizeRegexError(error);
ctx.addIssue({
code: "custom",
path: [field],
message:
- `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summarizeRegexError(error)}. ` +
- `${field} patterns use Rust regex (RE2-style) syntax.${hint}`,
+ `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summary}. ` +
+ `${field} patterns use Rust regex (RE2-style) syntax.${regexErrorHint(summary)}`,
});
}
}
@@ -36,3 +62,20 @@ function summarizeRegexError(error: string): string {
.find(l => l.startsWith("error:"));
return (line ?? error.split("\n")[0] ?? error).replace(/^error:\s*/, "");
}
+
+// The engine's own error already names the failing construct, so only add an
+// actionable hint for the errors where the fix is not obvious from the message.
+// Takes the summarized `error:` line, not the full diagnostic, which quotes the
+// user's pattern and could otherwise trigger a hint by containing these words.
+function regexErrorHint(summary: string): string {
+ if (/look-around|look-ahead|look-behind|backreference/i.test(summary)) {
+ return " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead.";
+ }
+ if (/exceeds size limit/i.test(summary)) {
+ return " The pattern expands to too many states when compiled, usually because of large or stacked counted repetitions such as {1000} or {5}{5}{5}. Lower the counts or use unbounded quantifiers like + and * instead.";
+ }
+ if (/Unicode not allowed/i.test(summary)) {
+ return " Patterns are matched against percent-encoded ASCII URLs, so Unicode-only constructs such as \\p{..} classes or non-ASCII characters inside [...] can never match. Remove them or match the percent-encoded form instead.";
+ }
+ return "";
+}