Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): reject includePaths/excludePaths regexes the engine can't honour (#4531)
firecrawl-spring[bot]cc6a0ad5c47f
diff --git a/apps/api/native/src/crawler.rs b/apps/api/native/src/crawler.rsindex 350395791..bd823b7ed 100644--- a/apps/api/native/src/crawler.rs+++ b/apps/api/native/src/crawler.rs@@ -45,6 +45,13 @@ pub struct FilterLinksResult {pub denial_reasons: HashMap<String, String>,}+#[derive(Serialize)]+#[napi(object)]+pub struct RegexValidationError {+ pub pattern: String,+ pub error: String,+}+#[derive(Deserialize)]#[napi(object)]pub struct FilterUrlCall {@@ -448,6 +455,24 @@ pub async fn filter_links(data: FilterLinksCall) -> Result<FilterLinksResult> {res.map_err(|e| Error::new(Status::GenericFailure, format!("Filter links error: {e}")))}+/// Validate regex patterns against the engine's regex flavor (the Rust `regex`+/// crate), which is what link filtering compiles them with. Returns an entry for+/// each pattern that fails to compile so callers can reject unsupported syntax+/// (e.g. look-around, backreferences) instead of silently ignoring it.+#[napi]+pub fn validate_regexes(patterns: Vec<String>) -> Vec<RegexValidationError> {+ patterns+ .into_iter()+ .filter_map(|pattern| match Regex::new(&pattern) {+ Ok(_) => None,+ Err(e) => Some(RegexValidationError {+ pattern,+ error: e.to_string(),+ }),+ })+ .collect()+}+fn _filter_url(data: FilterUrlCall) -> std::result::Result<FilterUrlResult, String> {let mut full_url = data.href.clone();diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 8831f958b..765ea4490 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -529,6 +529,17 @@ describe("V1 Types Validation", () => {expect(result.scrapeOptions).toBeDefined();expect(result.scrapeOptions.formats).toEqual(["markdown"]);});++ it("should reject path patterns using a negative lookahead", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/?(?!blog|works-with)[^/]+/.+"],+ }),+ ).toThrow(+ /look-around, including look-ahead and look-behind, is not supported/,+ );+ });});describe("mapRequestSchema", () => {diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 113747c94..40bf6c918 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -909,6 +909,67 @@ describe("V2 Types Validation", () => {expect(result.sitemap).toBe("only");});++ it("should accept anchored and substring path patterns", () => {+ const result = crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/?docs(/.*)?$", "/admin"],+ includePaths: ["^/blog"],+ });++ expect(result.excludePaths).toEqual(["^/?docs(/.*)?$", "/admin"]);+ expect(result.includePaths).toEqual(["^/blog"]);+ });++ it("should reject excludePaths patterns using a negative lookahead", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/?(?!blog|works-with)[^/]+/.+"],+ }),+ ).toThrow(+ /look-around, including look-ahead and look-behind, is not supported/,+ );+ });++ it("should reject includePaths patterns using a backreference", () => {+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: ["(a)\\1"],+ }),+ ).toThrow(/backreferences/);+ });++ it("should report the real error without the look-around hint for unrelated syntax errors", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["[abc"],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message).toMatch(/unclosed character class/);+ expect(message).not.toMatch(/Rewrite the pattern/);+ });++ it("should state the look-around limitation once and add a rewrite hint", () => {+ let message = "";+ try {+ crawlRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/?(?!blog)[^/]+/.+"],+ });+ } catch (e) {+ message = String(e);+ }++ expect(message.match(/not supported/g)).toHaveLength(1);+ expect(message).toMatch(/Rewrite the pattern/);+ });});describe("mapRequestSchema", () => {@@ -965,6 +1026,17 @@ describe("V2 Types Validation", () => {const result = mapRequestSchema.parse(input);expect(result.sitemap).toBe("only");});++ it("should reject path patterns the engine cannot honour", () => {+ expect(() =>+ mapRequestSchema.parse({+ url: "https://example.com",+ excludePaths: ["^/?(?!blog)[^/]+/.+"],+ }),+ ).toThrow(+ /look-around, including look-ahead and look-behind, is not supported/,+ );+ });});describe("batchScrapeRequestSchema", () => {diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex bc81ba97d..1b1b5380c 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -4,6 +4,7 @@ import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";+import { addPathRegexIssues } from "../../lib/crawl-regex";import type { PdfPageBlocks } from "../../scraper/scrapeURL/engines/pdf/types";import {ExtractorOptions,@@ -922,6 +923,10 @@ const crawlRequestSchemaBase = crawlerOptions.extend({export const crawlRequestSchema = crawlRequestSchemaBase.strict()+ .superRefine((x, ctx) => {+ addPathRegexIssues(x.includePaths, "includePaths", ctx);+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ }).refine(x => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),extractRefineOpts,@@ -1000,7 +1005,12 @@ const mapRequestSchemaBase = crawlerOptionsauditMetadata: auditMetadataSchema.optional(),});-export const mapRequestSchema = mapRequestSchemaBase.strict();+export const mapRequestSchema = mapRequestSchemaBase+ .strict()+ .superRefine((x, ctx) => {+ addPathRegexIssues(x.includePaths, "includePaths", ctx);+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ });// export type MapRequest = {// url: string;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 366049fbb..7c86c1ba1 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -5,6 +5,7 @@ import { protocolIncluded, checkUrl } from "../../lib/validateUrl";import { hasReachableHost } from "../../lib/url-utils";import { countries } from "../../lib/validate-country";import { includesFormat } from "../../lib/format-utils";+import { addPathRegexIssues } from "../../lib/crawl-regex";import {ExtractorOptions,PageOptions,@@ -1315,6 +1316,10 @@ const crawlRequestSchemaBase = crawlerOptions.extend({});export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase)+ .superRefine((x, ctx) => {+ addPathRegexIssues(x.includePaths, "includePaths", ctx);+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ }).refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {const scrapeOptionsValue = x.scrapeOptions ?? baseScrapeOptions.parse({});@@ -1364,7 +1369,12 @@ const mapRequestSchemaBase = crawlerOptionsauditMetadata: auditMetadataSchema.optional(),});-export const mapRequestSchema = strictWithMessage(mapRequestSchemaBase);+export const mapRequestSchema = strictWithMessage(+ mapRequestSchemaBase,+).superRefine((x, ctx) => {+ addPathRegexIssues(x.includePaths, "includePaths", ctx);+ addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+});// export type MapRequest = {// url: string;diff --git a/apps/api/src/lib/crawl-regex.ts b/apps/api/src/lib/crawl-regex.tsnew file mode 100644index 000000000..0efda29a9--- /dev/null+++ b/apps/api/src/lib/crawl-regex.ts@@ -0,0 +1,38 @@+import { validateRegexes } from "@mendable/firecrawl-rs";+import { z } from "zod";++// Link filtering compiles includePaths/excludePaths with the Rust `regex` crate+// (RE2-style: no look-around or backreferences). Historically an unsupported+// pattern compiled fine in most clients' regex flavor but was silently dropped+// by the engine, so the paths it was meant to filter got crawled anyway. Reject+// such patterns up front with a message that points at the actual limitation.+export function addPathRegexIssues(+ patterns: string[] | undefined,+ field: "includePaths" | "excludePaths",+ ctx: z.RefinementCtx,+): void {+ if (!patterns || patterns.length === 0) return;+ for (const { pattern, error } of validateRegexes(patterns)) {+ // The engine's own error already names look-around/backreferences as+ // unsupported, so only add the actionable rewrite hint in that case.+ const hint = /look-around|look-ahead|look-behind|backreference/i.test(error)+ ? " Rewrite the pattern using only constructs the engine supports, for example by listing the paths to keep in includePaths instead."+ : "";+ ctx.addIssue({+ code: "custom",+ path: [field],+ message:+ `Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summarizeRegexError(error)}. ` ++ `${field} patterns use Rust regex (RE2-style) syntax.${hint}`,+ });+ }+}++function summarizeRegexError(error: string): string {+ const line = error+ .split("\n")+ .map(l => l.trim())+ .reverse()+ .find(l => l.startsWith("error:"));+ return (line ?? error.split("\n")[0] ?? error).replace(/^error:\s*/, "");+}