Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): bound the cost of compiling includePaths/excludePaths patterns (#4533)
Gaurav Chadhaba5442250215
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex bd823b7ed..40fc457fa 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -305,15 +305,15 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultUrl::parse(&data.initial_url).map_err(|e| format!("Initial URL parse error: {e}"))?;let scope = crawl_scope(&initial_url);- let excludes_regex: Vec<Regex> = data+ let excludes_regex: Vec<PathRegex> = data.excludes.iter()- .filter_map(|e| Regex::new(e).ok())+ .filter_map(|e| compile_path_regex(e).ok()).collect();- let includes_regex: Vec<Regex> = data+ let includes_regex: Vec<PathRegex> = data.includes.iter()- .filter_map(|i| Regex::new(i).ok())+ .filter_map(|i| compile_path_regex(i).ok()).collect();let robot = build_robot(@@ -374,12 +374,20 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultpath};- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(match_target)) {+ if !excludes_regex.is_empty()+ && excludes_regex+ .iter()+ .any(|r| r.is_match(match_target.as_bytes()))+ {denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());continue;}- if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {+ if !includes_regex.is_empty()+ && !includes_regex+ .iter()+ .any(|r| r.is_match(match_target.as_bytes()))+ {denial_reasons.insert(link, INCLUDE_PATTERN.to_string());continue;}@@ -399,7 +407,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResultcontinue;}- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {+ if !excludes_regex.is_empty()+ && excludes_regex+ .iter()+ .any(|r| r.is_match(url_str.as_bytes()))+ {denial_reasons.insert(link, EXCLUDE_PATTERN.to_string());continue;}@@ -422,7 +434,11 @@ fn _filter_links(data: FilterLinksCall) -> std::result::Result<FilterLinksResult} else {path};- if !includes_regex.is_empty() && !includes_regex.iter().any(|r| r.is_match(match_target)) {+ if !includes_regex.is_empty()+ && !includes_regex+ .iter()+ .any(|r| r.is_match(match_target.as_bytes()))+ {denial_reasons.insert(link, INCLUDE_PATTERN.to_string());continue;}@@ -455,6 +471,47 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))}+/// Compiled form of a client-supplied includePaths/excludePaths pattern.+///+/// Path patterns are compiled with Unicode mode disabled, over bytes. Every+/// haystack they are matched against is a serialized `url::Url` path or href,+/// which is always percent-encoded ASCII, so ASCII and Unicode mode produce the+/// same matches. Unicode mode is what makes compilation expensive: a Unicode+/// `\w` is hundreds of codepoint ranges, so `\w{50}` alone compiles to more than+/// two megabytes of NFA, and `.{2000}` or `[\w-]{1,100}` several times over hit+/// the crate's 10 MiB default. In ASCII mode the same patterns compile in well+/// under a millisecond, which is what lets the size limit below be tight.+type PathRegex = regex::bytes::Regex;++/// Approximate upper bound, in bytes, on the compiled NFA of a single path+/// pattern. Patterns are untrusted, and the `regex` crate's compile time and+/// heap usage scale with the *compiled* size rather than the pattern's length:+/// `a{5}{5}{5}{5}{5}{5}` is 19 characters but expands to `a{15625}`. The crate's+/// default of 10 MiB lets a short pattern cost tens of milliseconds of CPU per+/// compile; this limit keeps it well under one millisecond while still admitting+/// path filters far larger than any realistic one.+/// https://docs.rs/regex/latest/regex/#untrusted-patterns+const PATH_REGEX_SIZE_LIMIT: usize = 256 * 1024;++/// Cache capacity of the lazy DFA used when matching a path pattern. Bounded so+/// a large pattern cannot claim the crate's 2 MiB default per compiled regex.+const PATH_REGEX_DFA_SIZE_LIMIT: usize = 256 * 1024;++/// Maximum nesting depth of a path pattern's syntax tree (crate default: 250).+const PATH_REGEX_NEST_LIMIT: u32 = 50;++/// Compile an includePaths/excludePaths pattern with resource limits suitable+/// for untrusted input. Every place that compiles a client-supplied path pattern+/// must go through this so validation and filtering agree on what is accepted.+fn compile_path_regex(pattern: &str) -> std::result::Result<PathRegex, regex::Error> {+ regex::bytes::RegexBuilder::new(pattern)+ .unicode(false)+ .size_limit(PATH_REGEX_SIZE_LIMIT)+ .dfa_size_limit(PATH_REGEX_DFA_SIZE_LIMIT)+ .nest_limit(PATH_REGEX_NEST_LIMIT)+ .build()+}+/// Validate regex patterns against the engine's regex flavor (the Rust `regex`/// crate), which is what link filtering compiles them with. Returns an entry for/// each pattern that fails to compile so callers can reject unsupported syntax@@ -463,7 +520,7 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {pub fn validate_regexes(patterns: Vec<String>) -> Vec<RegexValidationError> {patterns.into_iter()- .filter_map(|pattern| match Regex::new(&pattern) {+ .filter_map(|pattern| match compile_path_regex(&pattern) {Ok(_) => None,Err(e) => Some(RegexValidationError {pattern,@@ -532,10 +589,10 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri});}- let excludes_regex: Vec<Regex> = data+ let excludes_regex: Vec<PathRegex> = data.excludes.iter()- .filter_map(|e| Regex::new(e).ok())+ .filter_map(|e| compile_path_regex(e).ok()).collect();let robot = build_robot(@@ -554,7 +611,7 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri});}- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path)) {+ if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(path.as_bytes())) {return Ok(FilterUrlResult {allowed: false,url: None,@@ -587,7 +644,11 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri});}- if !excludes_regex.is_empty() && excludes_regex.iter().any(|r| r.is_match(url_str)) {+ if !excludes_regex.is_empty()+ && excludes_regex+ .iter()+ .any(|r| r.is_match(url_str.as_bytes()))+ {return Ok(FilterUrlResult {allowed: false,url: None,@@ -613,7 +674,9 @@ fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, Stri// it), matched via the same PSL check used for allowSubdomains.if data.allow_external_content_links&& !is_external_main_page(url_str)- && (is_internal_link(&context_url, &base_url) || is_subdomain(&url, &context_url) || is_internal_link(&url, &context_url))+ && (is_internal_link(&context_url, &base_url)+ || is_subdomain(&url, &context_url)+ || is_internal_link(&url, &context_url)){return Ok(FilterUrlResult {allowed: true,diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 765ea4490..47fda3127 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -540,6 +540,24 @@ describe("V1 Types Validation", () => {/look-around, including look-ahead and look-behind, is not supported/,);});++ it("should reject patterns whose compiled form exceeds the size limit", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],+ }),+ ).toThrow(/exceeds size limit/);+ });++ it("should reject more than the maximum number of path patterns", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),+ }),+ ).toThrow(/at most 100 patterns/);+ });});describe("mapRequestSchema", () => {diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 367092fef..8d9f813ae 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -191,6 +191,47 @@ describe("Crawl tests", () => {10 * scrapeTimeout,);+ it.concurrent(+ "rejects path patterns that are too expensive to compile",+ async () => {+ // 19 characters that expand to a{15625}. Compiling patterns like this+ // with the engine's default limits is a CPU denial-of-service vector.+ const res = await crawlStart(+ {+ url: "https://firecrawl.dev",+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],+ },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toMatch(/exceeds size limit/);+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects more path patterns than the engine will compile",+ async () => {+ const res = await crawlStart(+ {+ url: "https://firecrawl.dev",+ excludePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),+ },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ // Schema-level (non-custom) issues are reported in details, not error.+ expect(+ res.body.details.map((issue: { message: string }) => issue.message),+ ).toContainEqual(expect.stringMatching(/at most 100 patterns/));+ },+ scrapeTimeout,+ );+// TODO: port to new dynamic url system// concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(// "filters URLs properly when using regexOnFullURL",diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 40bf6c918..60ebca9c6 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -1,4 +1,8 @@import { z } from "zod";+import {+ MAX_PATH_PATTERNS,+ MAX_PATH_PATTERN_LENGTH,+} from "../../../lib/crawl-regex";import {scrapeRequestSchema,parseRequestSchema,@@ -970,6 +974,105 @@ describe("V2 Types Validation", () => {expect(message.match(/not supported/g)).toHaveLength(1);expect(message).toMatch(/Rewrite the pattern/);});++ it("should accept counted repetitions of word classes that real path filters use", () => {+ // Unicode \w is hundreds of ranges, so these used to exceed the engine's+ // compiled-size limit and were silently dropped. Path haystacks are+ // percent-encoded ASCII, so they are compiled in ASCII mode and are cheap.+ const result = crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: [+ "^/[\\w-]{1,100}/[\\w-]{1,100}/[\\w-]{1,100}/?$",+ "\\w{300}",+ ],+ });++ expect(result.includePaths).toHaveLength(2);+ });++ it("should reject patterns whose compiled form exceeds the size limit", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["a{5}{5}{5}{5}{5}{5}"],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message).toMatch(/exceeds size limit/);+ expect(message).toMatch(/stacked counted repetitions/);+ });++ it("should reject Unicode-only constructs with a hint", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/\\p{Greek}+"],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message).toMatch(/Unicode not allowed/);+ expect(message).toMatch(/percent-encoded ASCII/);+ });++ it("should reject more than the maximum number of path patterns", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: Array.from(+ { length: MAX_PATH_PATTERNS + 1 },+ (_, i) => `^/p${i}`,+ ),+ }),+ ).toThrow(/at most 100 patterns/);+ });++ it("should not compile patterns once the count cap is exceeded", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: [+ ...Array.from({ length: MAX_PATH_PATTERNS }, (_, i) => `^/p${i}`),+ "[abc",+ ],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message).toMatch(/at most 100 patterns/);+ expect(message).not.toMatch(/unclosed character class/);+ });++ it("should not derive hints from user pattern text", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["[exceeds size limit"],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message).toMatch(/unclosed character class/);+ expect(message).not.toMatch(/stacked counted repetitions/);+ });++ it("should reject path patterns longer than the maximum length", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: ["^/" + "a".repeat(MAX_PATH_PATTERN_LENGTH)],+ }),+ ).toThrow(/at most 2000 characters/);+ });});describe("mapRequestSchema", () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 1b1b5380c..291a02729 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -4,7 +4,7 @@ import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";-import { addPathRegexIssues } from "../../lib/crawl-regex";+import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";import {ExtractorOptions,@@ -873,8 +873,8 @@ export type BatchScrapeRequest = z.infer<typeof batchScrapeRequestSchema>;export type BatchScrapeRequestInput = z.input<typeof batchScrapeRequestSchema>;const crawlerOptions = z.strictObject({- includePaths: z.string().array().prefault([]),- excludePaths: z.string().array().prefault([]),+ includePaths: pathPatternsSchema.prefault([]),+ excludePaths: pathPatternsSchema.prefault([]),maxDepth: z.number().prefault(10), // default?maxDiscoveryDepth: z.number().optional(),limit: z.number().prefault(10000), // default?diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 7c86c1ba1..3f4c717e0 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -5,7 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";import { includesFormat } from "../../lib/format-utils";-import { addPathRegexIssues } from "../../lib/crawl-regex";+import { addPathRegexIssues, pathPatternsSchema } from "../../lib/crawl-regex";import {ExtractorOptions,PageOptions,@@ -1268,8 +1268,8 @@ export type BatchScrapeRequestInput = Omit<};export const crawlerOptions = z.strictObject({- includePaths: z.string().array().prefault([]),- excludePaths: z.string().array().prefault([]),+ includePaths: pathPatternsSchema.prefault([]),+ excludePaths: pathPatternsSchema.prefault([]),maxDiscoveryDepth: z.number().optional(),limit: z.number().prefault(10000), // default?crawlEntireDomain: z.boolean().optional(),diff --git a/apps/api/src/lib/crawl-regex.ts b/apps/api/src/lib/crawl-regex.tsindex 0efda29a9..c811bfa5b 100644--- a/apps/api/src/lib/crawl-regex.ts+++ b/apps/api/src/lib/crawl-regex.ts@@ -1,6 +1,26 @@import { validateRegexes } from "@mendable/firecrawl-rs";import { z } from "zod";+// Every pattern is compiled by the engine at request time (validation) and again+// when links are filtered, so the amount of work a single request can demand is+// count x per-pattern cost. The engine bounds per-pattern cost (see+// compile_path_regex in native/src/crawler.rs); these bound the count and the+// pattern length, which is what parsing cost scales with.+export const MAX_PATH_PATTERNS = 100;+export const MAX_PATH_PATTERN_LENGTH = 2000;++export const pathPatternsSchema = z+ .string()+ .max(+ MAX_PATH_PATTERN_LENGTH,+ `Each includePaths/excludePaths pattern must be at most ${MAX_PATH_PATTERN_LENGTH} characters.`,+ )+ .array()+ .max(+ MAX_PATH_PATTERNS,+ `includePaths and excludePaths each accept at most ${MAX_PATH_PATTERNS} patterns.`,+ );+// Link filtering compiles includePaths/excludePaths with the Rust `regex` crate// (RE2-style: no look-around or backreferences). Historically an unsupported// pattern compiled fine in most clients' regex flavor but was silently dropped@@ -12,18 +32,24 @@ export function addPathRegexIssues(ctx: z.RefinementCtx,): void {if (!patterns || patterns.length === 0) return;+ // Zod still runs this refinement when pathPatternsSchema has already reported+ // a count or length violation, so re-check the caps here: they are what bound+ // the total compile work a request can demand, and an over-cap request has+ // already been rejected.+ if (+ patterns.length > MAX_PATH_PATTERNS ||+ patterns.some(p => p.length > MAX_PATH_PATTERN_LENGTH)+ ) {+ return;+ }for (const { pattern, error } of validateRegexes(patterns)) {- // The engine's own error already names look-around/backreferences as- // unsupported, so only add the actionable rewrite hint in that case.- const hint = /look-around|look-ahead|look-behind|backreference/i.test(error)- ? " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead."- : "";+ const summary = summarizeRegexError(error);ctx.addIssue({code: "custom",path: [field],message:- `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summarizeRegexError(error)}. ` +- `${field} patterns use Rust regex (RE2-style) syntax.${hint}`,+ `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summary}. ` ++ `${field} patterns use Rust regex (RE2-style) syntax.${regexErrorHint(summary)}`,});}}@@ -36,3 +62,20 @@ function summarizeRegexError(error: string): string {.find(l => l.startsWith("error:"));return (line ?? error.split("\n")[0] ?? error).replace(/^error:\s*/, "");}++// The engine's own error already names the failing construct, so only add an+// actionable hint for the errors where the fix is not obvious from the message.+// Takes the summarized `error:` line, not the full diagnostic, which quotes the+// user's pattern and could otherwise trigger a hint by containing these words.+function regexErrorHint(summary: string): string {+ if (/look-around|look-ahead|look-behind|backreference/i.test(summary)) {+ return " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead.";+ }+ if (/exceeds size limit/i.test(summary)) {+ return " The pattern expands to too many states when compiled, usually because of large or stacked counted repetitions such as {1000} or {5}{5}{5}. Lower the counts or use unbounded quantifiers like + and * instead.";+ }+ if (/Unicode not allowed/i.test(summary)) {+ return " Patterns are matched against percent-encoded ASCII URLs, so Unicode-only constructs such as \\p{..} classes or non-ASCII characters inside [...] can never match. Remove them or match the percent-encoded form instead.";+ }+ return "";+}