Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): reject includePaths/excludePaths regexes the engine can't honour (#4531)

firecrawl-spring[bot]cc6a0ad5c47f
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rs
index 350395791..bd823b7ed 100644
--- a/apps/api/native/src/crawler.rs
+++ b/apps/api/native/src/crawler.rs
@@ -45,6 +45,13 @@ pub struct FilterLinksResult {
pub denial_reasons: HashMap<String, String>,
}
+#[derive(Serialize)]
+#[napi(object)]
+pub struct RegexValidationError {
+ pub pattern: String,
+ pub error: String,
+}
+
#[derive(Deserialize)]
#[napi(object)]
pub struct FilterUrlCall {
@@ -448,6 +455,24 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {
res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))
}
+/// Validate regex patterns against the engine's regex flavor (the Rust `regex`
+/// crate), which is what link filtering compiles them with. Returns an entry for
+/// each pattern that fails to compile so callers can reject unsupported syntax
+/// (e.g. look-around, backreferences) instead of silently ignoring it.
+#[napi]
+pub fn validate_regexes(patterns: Vec<String>) -> Vec<RegexValidationError> {
+ patterns
+ .into_iter()
+ .filter_map(|pattern| match Regex::new(&pattern) {
+ Ok(_) => None,
+ Err(e) => Some(RegexValidationError {
+ pattern,
+ error: e.to_string(),
+ }),
+ })
+ .collect()
+}
+
fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {
let mut full_url = data.href.clone();
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
index 8831f958b..765ea4490 100644
--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts
@@ -529,6 +529,17 @@ describe("V1 Types Validation", () => {
expect(result.scrapeOptions).toBeDefined();
expect(result.scrapeOptions.formats).toEqual(["markdown"]);
});
+
+ it("should reject path patterns using a negative lookahead", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/?(?!blog|works-with)[^/]+/.+"],
+ }),
+ ).toThrow(
+ /look-around, including look-ahead and look-behind, is not supported/,
+ );
+ });
});
describe("mapRequestSchema", () => {
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index 113747c94..40bf6c918 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -909,6 +909,67 @@ describe("V2 Types Validation", () => {
expect(result.sitemap).toBe("only");
});
+
+ it("should accept anchored and substring path patterns", () => {
+ const result = crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/?docs(/.*)?$", "/admin"],
+ includePaths: ["^/blog"],
+ });
+
+ expect(result.excludePaths).toEqual(["^/?docs(/.*)?$", "/admin"]);
+ expect(result.includePaths).toEqual(["^/blog"]);
+ });
+
+ it("should reject excludePaths patterns using a negative lookahead", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/?(?!blog|works-with)[^/]+/.+"],
+ }),
+ ).toThrow(
+ /look-around, including look-ahead and look-behind, is not supported/,
+ );
+ });
+
+ it("should reject includePaths patterns using a backreference", () => {
+ expect(() =>
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ includePaths: ["(a)\\1"],
+ }),
+ ).toThrow(/backreferences/);
+ });
+
+ it("should report the real error without the look-around hint for unrelated syntax errors", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["[abc"],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message).toMatch(/unclosed character class/);
+ expect(message).not.toMatch(/Rewrite the pattern/);
+ });
+
+ it("should state the look-around limitation once and add a rewrite hint", () => {
+ let message = "";
+ try {
+ crawlRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/?(?!blog)[^/]+/.+"],
+ });
+ } catch (e) {
+ message = String(e);
+ }
+
+ expect(message.match(/not supported/g)).toHaveLength(1);
+ expect(message).toMatch(/Rewrite the pattern/);
+ });
});
describe("mapRequestSchema", () => {
@@ -965,6 +1026,17 @@ describe("V2 Types Validation", () => {
const result = mapRequestSchema.parse(input);
expect(result.sitemap).toBe("only");
});
+
+ it("should reject path patterns the engine cannot honour", () => {
+ expect(() =>
+ mapRequestSchema.parse({
+ url: "https://example.com",
+ excludePaths: ["^/?(?!blog)[^/]+/.+"],
+ }),
+ ).toThrow(
+ /look-around, including look-ahead and look-behind, is not supported/,
+ );
+ });
});
describe("batchScrapeRequestSchema", () => {
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index bc81ba97d..1b1b5380c 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -4,6 +4,7 @@ import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
+import { addPathRegexIssues } from "../../lib/crawl-regex";
import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";
import {
ExtractorOptions,
@@ -922,6 +923,10 @@ const crawlRequestSchemaBase = crawlerOptions.extend({
export const crawlRequestSchema = crawlRequestSchemaBase
.strict()
+ .superRefine((x, ctx) => {
+ addPathRegexIssues(x.includePaths, "includePaths", ctx);
+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);
+ })
.refine(
x => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),
extractRefineOpts,
@@ -1000,7 +1005,12 @@ const mapRequestSchemaBase = crawlerOptions
auditMetadata: auditMetadataSchema.optional(),
});
-export const mapRequestSchema = mapRequestSchemaBase.strict();
+export const mapRequestSchema = mapRequestSchemaBase
+ .strict()
+ .superRefine((x, ctx) => {
+ addPathRegexIssues(x.includePaths, "includePaths", ctx);
+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);
+ });
// export type MapRequest = {
// url: string;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 366049fbb..7c86c1ba1 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -5,6 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
import { hasReachableHost } from "../../lib/url-utils";
import { countries } from "../../lib/validate-country";
import { includesFormat } from "../../lib/format-utils";
+import { addPathRegexIssues } from "../../lib/crawl-regex";
import {
ExtractorOptions,
PageOptions,
@@ -1315,6 +1316,10 @@ const crawlRequestSchemaBase = crawlerOptions.extend({
});
export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase)
+ .superRefine((x, ctx) => {
+ addPathRegexIssues(x.includePaths, "includePaths", ctx);
+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);
+ })
.refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform(x => {
const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});
@@ -1364,7 +1369,12 @@ const mapRequestSchemaBase = crawlerOptions
auditMetadata: auditMetadataSchema.optional(),
});
-export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);
+export const mapRequestSchema = strictWithMessage(
+ mapRequestSchemaBase,
+).superRefine((x, ctx) => {
+ addPathRegexIssues(x.includePaths, "includePaths", ctx);
+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);
+});
// export type MapRequest = {
// url: string;
diff --git a/apps/api/src/lib/crawl-regex.ts b/apps/api/src/lib/crawl-regex.ts
new file mode 100644
index 000000000..0efda29a9
--- /dev/null
+++ b/apps/api/src/lib/crawl-regex.ts
@@ -0,0 +1,38 @@
+import { validateRegexes } from "@mendable/firecrawl-rs";
+import { z } from "zod";
+
+// Link filtering compiles includePaths/excludePaths with the Rust `regex` crate
+// (RE2-style: no look-around or backreferences). Historically an unsupported
+// pattern compiled fine in most clients' regex flavor but was silently dropped
+// by the engine, so the paths it was meant to filter got crawled anyway. Reject
+// such patterns up front with a message that points at the actual limitation.
+export function addPathRegexIssues(
+ patterns: string[] | undefined,
+ field: "includePaths" | "excludePaths",
+ ctx: z.RefinementCtx,
+): void {
+ if (!patterns || patterns.length === 0) return;
+ for (const { pattern, error } of validateRegexes(patterns)) {
+ // The engine's own error already names look-around/backreferences as
+ // unsupported, so only add the actionable rewrite hint in that case.
+ const hint = /look-around|look-ahead|look-behind|backreference/i.test(error)
+ ? " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead."
+ : "";
+ ctx.addIssue({
+ code: "custom",
+ path: [field],
+ message:
+ `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summarizeRegexError(error)}. ` +
+ `${field} patterns use Rust regex (RE2-style) syntax.${hint}`,
+ });
+ }
+}
+
+function summarizeRegexError(error: string): string {
+ const line = error
+ .split("\n")
+ .map(l => l.trim())
+ .reverse()
+ .find(l => l.startsWith("error:"));
+ return (line ?? error.split("\n")[0] ?? error).replace(/^error:\s*/, "");
+}