Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): raise includePaths/excludePaths cap from 100 to 1000 patterns (#4611)
Gaurav Chadhada16e7935022
diff --git a/apps/api/src/__tests__/snips/v1/types-validation.test.ts b/apps/api/src/__tests__/snips/v1/types-validation.test.tsindex 47fda3127..26f52dc24 100644--- a/apps/api/src/__tests__/snips/v1/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v1/types-validation.test.ts@@ -1,4 +1,10 @@import { z } from "zod";+import {+ MAX_PATH_PATTERNS,+ MAX_PATH_PATTERN_LENGTH,+ MAX_TOTAL_PATH_PATTERNS,+ MAX_TOTAL_PATH_PATTERN_CHARS,+} from "../../../lib/crawl-regex";import {scrapeRequestSchema,scrapeOptions,@@ -550,13 +556,59 @@ describe("V1 Types Validation", () => {).toThrow(/exceeds size limit/);});+ it("should accept several hundred keyword patterns per field", () => {+ const result = crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: 300 }, (_, i) => `topic${i}`),+ excludePaths: Array.from({ length: 300 }, (_, i) => `skip${i}`),+ });+ expect(result.includePaths).toHaveLength(300);+ expect(result.excludePaths).toHaveLength(300);+ });+it("should reject more than the maximum number of path patterns", () => {expect(() =>crawlRequestSchema.parse({url: "https://example.com",- includePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),+ includePaths: Array.from(+ { length: MAX_PATH_PATTERNS + 1 },+ (_, i) => `^/p${i}`,+ ),+ }),+ ).toThrow(new RegExp(`at most ${MAX_PATH_PATTERNS} patterns`));+ });++ it("should reject more than the aggregate number of path patterns", () => {+ // Each field is within its own cap, but together they exceed the budget.+ const half = Math.floor(MAX_TOTAL_PATH_PATTERNS / 2) + 1;+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: half }, (_, i) => `^/a${i}`),+ excludePaths: Array.from({ length: half }, (_, i) => `^/b${i}`),}),- ).toThrow(/at most 100 patterns/);+ ).toThrow(+ new RegExp(+ `together accept at most ${MAX_TOTAL_PATH_PATTERNS} patterns`,+ ),+ );+ });++ it("should reject path patterns exceeding the aggregate character budget", () => {+ const pattern = "^/" + "a".repeat(MAX_PATH_PATTERN_LENGTH - 2);+ const perField =+ Math.floor(MAX_TOTAL_PATH_PATTERN_CHARS / pattern.length / 2) + 1;+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: perField }, () => pattern),+ excludePaths: Array.from({ length: perField }, () => pattern),+ }),+ ).toThrow(+ new RegExp(+ `together accept at most ${MAX_TOTAL_PATH_PATTERN_CHARS} characters`,+ ),+ );});});diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 8d9f813ae..63e0c2d6a 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -20,6 +20,10 @@ import {TEST_API_URL,} from "./lib";import request from "./lib";+import {+ MAX_PATH_PATTERNS,+ MAX_TOTAL_PATH_PATTERNS,+} from "../../../lib/crawl-regex";import { describe, it, expect } from "vitest";let identity: Identity;@@ -211,13 +215,36 @@ describe("Crawl tests", () => {scrapeTimeout,);+ it.concurrent(+ "accepts several hundred keyword path patterns per field",+ async () => {+ const res = await crawlStart(+ {+ url: "https://firecrawl.dev",+ limit: 1,+ includePaths: Array.from({ length: 300 }, (_, i) => `topic${i}`),+ excludePaths: Array.from({ length: 300 }, (_, i) => `skip${i}`),+ },+ identity,+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(typeof res.body.id).toBe("string");+ },+ scrapeTimeout,+ );+it.concurrent("rejects more path patterns than the engine will compile",async () => {const res = await crawlStart({url: "https://firecrawl.dev",- excludePaths: Array.from({ length: 101 }, (_, i) => `^/p${i}`),+ excludePaths: Array.from(+ { length: MAX_PATH_PATTERNS + 1 },+ (_, i) => `^/p${i}`,+ ),},identity,);@@ -227,7 +254,35 @@ describe("Crawl tests", () => {// Schema-level (non-custom) issues are reported in details, not error.expect(res.body.details.map((issue: { message: string }) => issue.message),- ).toContainEqual(expect.stringMatching(/at most 100 patterns/));+ ).toContainEqual(+ expect.stringMatching(+ new RegExp(`at most ${MAX_PATH_PATTERNS} patterns`),+ ),+ );+ },+ scrapeTimeout,+ );++ it.concurrent(+ "rejects include and exclude patterns that together exceed the budget",+ async () => {+ const half = Math.floor(MAX_TOTAL_PATH_PATTERNS / 2) + 1;+ const res = await crawlStart(+ {+ url: "https://firecrawl.dev",+ includePaths: Array.from({ length: half }, (_, i) => `^/a${i}`),+ excludePaths: Array.from({ length: half }, (_, i) => `^/b${i}`),+ },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toMatch(+ new RegExp(+ `together accept at most ${MAX_TOTAL_PATH_PATTERNS} patterns`,+ ),+ );},scrapeTimeout,);diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex 60ebca9c6..1088495f3 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -2,6 +2,9 @@ import { z } from "zod";import {MAX_PATH_PATTERNS,MAX_PATH_PATTERN_LENGTH,+ MAX_TOTAL_PATH_PATTERNS,+ MAX_TOTAL_PATH_PATTERN_CHARS,+ collectPathPatternIssues,} from "../../../lib/crawl-regex";import {scrapeRequestSchema,@@ -1020,6 +1023,18 @@ describe("V2 Types Validation", () => {expect(message).toMatch(/percent-encoded ASCII/);});+ it("should accept several hundred keyword patterns per field", () => {+ // Keyword-based filtering sends one short pattern per term; a few+ // hundred per field must not be rejected by the count cap.+ const result = crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: 300 }, (_, i) => `topic${i}`),+ excludePaths: Array.from({ length: 300 }, (_, i) => `skip${i}`),+ });+ expect(result.includePaths).toHaveLength(300);+ expect(result.excludePaths).toHaveLength(300);+ });+it("should reject more than the maximum number of path patterns", () => {expect(() =>crawlRequestSchema.parse({@@ -1029,7 +1044,62 @@ describe("V2 Types Validation", () => {(_, i) => `^/p${i}`,),}),- ).toThrow(/at most 100 patterns/);+ ).toThrow(new RegExp(`at most ${MAX_PATH_PATTERNS} patterns`));+ });++ it("should reject more than the aggregate number of path patterns", () => {+ // Each field is within its own cap, but together they exceed the budget.+ const half = Math.floor(MAX_TOTAL_PATH_PATTERNS / 2) + 1;+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: half }, (_, i) => `^/a${i}`),+ excludePaths: Array.from({ length: half }, (_, i) => `^/b${i}`),+ }),+ ).toThrow(+ new RegExp(+ `together accept at most ${MAX_TOTAL_PATH_PATTERNS} patterns`,+ ),+ );+ });++ it("should report aggregate budget issues at the request root", () => {+ // excludePaths sits exactly at its own cap; one includePaths pattern+ // pushes the total over the request-wide budget.+ const result = crawlRequestSchema.safeParse({+ url: "https://example.com",+ excludePaths: Array.from(+ { length: MAX_PATH_PATTERNS },+ (_, i) => `^/b${i}`,+ ),+ includePaths: ["^/a"],+ });+ expect(result.success).toBe(false);+ if (!result.success) {+ const issue = result.error.issues.find(i =>+ /together accept at most/.test(i.message),+ );+ expect(issue).toBeDefined();+ // Neither field is individually at fault, so do not point at one.+ expect(issue!.path).toEqual([]);+ }+ });++ it("should reject path patterns exceeding the aggregate character budget", () => {+ const pattern = "^/" + "a".repeat(MAX_PATH_PATTERN_LENGTH - 2);+ const perField =+ Math.floor(MAX_TOTAL_PATH_PATTERN_CHARS / pattern.length / 2) + 1;+ expect(() =>+ crawlRequestSchema.parse({+ url: "https://example.com",+ includePaths: Array.from({ length: perField }, () => pattern),+ excludePaths: Array.from({ length: perField }, () => pattern),+ }),+ ).toThrow(+ new RegExp(+ `together accept at most ${MAX_TOTAL_PATH_PATTERN_CHARS} characters`,+ ),+ );});it("should not compile patterns once the count cap is exceeded", () => {@@ -1046,7 +1116,9 @@ describe("V2 Types Validation", () => {message = String(e);}- expect(message).toMatch(/at most 100 patterns/);+ expect(message).toMatch(+ new RegExp(`at most ${MAX_PATH_PATTERNS} patterns`),+ );expect(message).not.toMatch(/unclosed character class/);});@@ -1071,7 +1143,68 @@ describe("V2 Types Validation", () => {url: "https://example.com",includePaths: ["^/" + "a".repeat(MAX_PATH_PATTERN_LENGTH)],}),- ).toThrow(/at most 2000 characters/);+ ).toThrow(new RegExp(`at most ${MAX_PATH_PATTERN_LENGTH} characters`));+ });+ });++ describe("collectPathPatternIssues", () => {+ // Options generated from a crawl prompt are merged after schema+ // validation, so the controller validates them with this helper directly.+ it("should accept merged options within every limit", () => {+ expect(+ collectPathPatternIssues({+ includePaths: Array.from({ length: 300 }, (_, i) => `topic${i}`),+ excludePaths: ["^/careers", "^/jobs"],+ }),+ ).toEqual([]);+ });++ it("should report malformed generated fields instead of throwing", () => {+ expect(+ collectPathPatternIssues({+ includePaths: "^/blog",+ excludePaths: ["^/jobs", 42],+ }),+ ).toEqual([+ {+ kind: "shape",+ path: ["includePaths"],+ message: "includePaths must be an array of strings.",+ },+ {+ kind: "shape",+ path: ["excludePaths"],+ message: "excludePaths must be an array of strings.",+ },+ ]);+ expect(collectPathPatternIssues({ includePaths: null })).toEqual([]);+ });++ it("should report per-field caps that the schema did not see", () => {+ const issues = collectPathPatternIssues({+ includePaths: Array.from(+ { length: MAX_PATH_PATTERNS + 1 },+ (_, i) => `^/p${i}`,+ ),+ });+ expect(issues).toHaveLength(1);+ expect(issues[0].kind).toBe("field-cap");+ expect(issues[0].path).toEqual(["includePaths"]);+ });++ it("should report the aggregate budget and unsupported syntax", () => {+ const half = Math.floor(MAX_TOTAL_PATH_PATTERNS / 2) + 1;+ const budget = collectPathPatternIssues({+ includePaths: Array.from({ length: half }, (_, i) => `^/a${i}`),+ excludePaths: Array.from({ length: half }, (_, i) => `^/b${i}`),+ });+ expect(budget.map(i => i.kind)).toEqual(["budget"]);++ const syntax = collectPathPatternIssues({+ excludePaths: ["^/ok", "(?<=a)b"],+ });+ expect(syntax.map(i => i.kind)).toEqual(["syntax"]);+ expect(syntax[0].message).toMatch(/look-around/);});});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 2381a32ec..a53698db0 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -924,8 +924,7 @@ const crawlRequestSchemaBase = crawlerOptions.extend({export const crawlRequestSchema = crawlRequestSchemaBase.strict().superRefine((x, ctx) => {- addPathRegexIssues(x.includePaths, "includePaths", ctx);- addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ addPathRegexIssues(x, ctx);}).refine(x => (x.scrapeOptions ? extractRefine(x.scrapeOptions) : true),@@ -1008,8 +1007,7 @@ const mapRequestSchemaBase = crawlerOptionsexport const mapRequestSchema = mapRequestSchemaBase.strict().superRefine((x, ctx) => {- addPathRegexIssues(x.includePaths, "includePaths", ctx);- addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ addPathRegexIssues(x, ctx);});// export type MapRequest = {diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 1ea7f880e..a927fe43c 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -25,6 +25,7 @@ import {formatTypesOf,} from "../../lib/key-restriction";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";+import { collectPathPatternIssues } from "../../lib/crawl-regex";import {crawlGroup,resolveNewGroupBackend,@@ -245,23 +246,15 @@ export async function crawlController(}}- if (Array.isArray(finalCrawlerOptions.includePaths)) {- for (const x of finalCrawlerOptions.includePaths) {- try {- new RegExp(x);- } catch (e) {- return res.status(400).json({ success: false, error: e.message });- }- }- }-- if (Array.isArray(finalCrawlerOptions.excludePaths)) {- for (const x of finalCrawlerOptions.excludePaths) {- try {- new RegExp(x);- } catch (e) {- return res.status(400).json({ success: false, error: e.message });- }+ // The request schema already validated user-supplied includePaths /+ // excludePaths. Options generated from a prompt bypass the schema, so hold+ // the merged result to the same caps, budget, and engine syntax.+ if (req.body.prompt) {+ const pathPatternIssues = collectPathPatternIssues(finalCrawlerOptions);+ if (pathPatternIssues.length > 0) {+ return res+ .status(400)+ .json({ success: false, error: pathPatternIssues[0].message });}}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 26bda62ca..9b0dd1024 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1318,8 +1318,7 @@ const crawlRequestSchemaBase = crawlerOptions.extend({export const crawlRequestSchema = strictWithMessage(crawlRequestSchemaBase).superRefine((x, ctx) => {- addPathRegexIssues(x.includePaths, "includePaths", ctx);- addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ addPathRegexIssues(x, ctx);}).refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {@@ -1373,8 +1372,7 @@ const mapRequestSchemaBase = crawlerOptionsexport const mapRequestSchema = strictWithMessage(mapRequestSchemaBase,).superRefine((x, ctx) => {- addPathRegexIssues(x.includePaths, "includePaths", ctx);- addPathRegexIssues(x.excludePaths, "excludePaths", ctx);+ addPathRegexIssues(x, ctx);});// export type MapRequest = {diff --git a/apps/api/src/lib/crawl-regex.ts b/apps/api/src/lib/crawl-regex.tsindex c811bfa5b..9a479900b 100644--- a/apps/api/src/lib/crawl-regex.ts+++ b/apps/api/src/lib/crawl-regex.ts@@ -6,9 +6,22 @@ import { z } from "zod";// count x per-pattern cost. The engine bounds per-pattern cost (see// compile_path_regex in native/src/crawler.rs); these bound the count and the// pattern length, which is what parsing cost scales with.-export const MAX_PATH_PATTERNS = 100;+//+// Keyword-style filtering (one short pattern per term) routinely needs a few+// hundred patterns per field, so the count cap has headroom for that. Measured+// against the native module, 1000 short keyword patterns compile in ~2 ms.+export const MAX_PATH_PATTERNS = 1000;export const MAX_PATH_PATTERN_LENGTH = 2000;+// includePaths and excludePaths are compiled together, so the per-field caps+// alone would let one request demand 2 x 1000 x 2000 characters of compile+// work. These bound the request as a whole: total pattern count and total+// pattern characters across both fields. The worst case that fits (1000+// patterns whose combined length is 100k characters) compiles in ~120 ms on the+// native module, versus ~700 ms for the unbudgeted per-field maximum.+export const MAX_TOTAL_PATH_PATTERNS = 1000;+export const MAX_TOTAL_PATH_PATTERN_CHARS = 100_000;+export const pathPatternsSchema = z.string().max(@@ -21,37 +34,144 @@ export const pathPatternsSchema = z`includePaths and excludePaths each accept at most ${MAX_PATH_PATTERNS} patterns.`,);+type PathPatternField = "includePaths" | "excludePaths";++// Values are `unknown` because the v2 crawl controller passes crawler options+// that were partly produced by an LLM from a `prompt`, which may return the+// wrong shape entirely (a string instead of an array, numbers in the array).+type PathPatternFields = {+ includePaths?: unknown;+ excludePaths?: unknown;+};++type PathPatternIssue = {+ // "shape": the field is not an array of strings.+ // "field-cap": a single field exceeds its count or length cap.+ // "budget": both fields together exceed the request-wide budget.+ // "syntax": a pattern does not compile with the engine.+ kind: "shape" | "field-cap" | "budget" | "syntax";+ // Empty for request-wide (cross-field) issues.+ path: PathPatternField[];+ message: string;+};++// Validate includePaths/excludePaths as a unit: the per-field caps, then the+// request-wide budget, then each pattern against the engine. Nothing is+// compiled once any cap is exceeded, because the caps are what bound the+// compile work a request can demand.+//+// This is the single source of truth for what path patterns are accepted. The+// request schemas call it through addPathRegexIssues; the v2 crawl controller+// calls it directly on the final crawler options, because includePaths /+// excludePaths generated from a crawl `prompt` are merged in after schema+// validation and must meet the same limits before they reach the crawler.+export function collectPathPatternIssues(+ fields: PathPatternFields,+): PathPatternIssue[] {+ const shapeIssues: PathPatternIssue[] = [];+ const asPatterns = (field: PathPatternField): string[] => {+ const value = fields[field];+ if (value === undefined || value === null) return [];+ if (Array.isArray(value) && value.every(p => typeof p === "string")) {+ return value;+ }+ shapeIssues.push({+ kind: "shape",+ path: [field],+ message: `${field} must be an array of strings.`,+ });+ return [];+ };+ const include = asPatterns("includePaths");+ const exclude = asPatterns("excludePaths");+ if (shapeIssues.length > 0) return shapeIssues;+ if (include.length === 0 && exclude.length === 0) return [];++ const capIssues: PathPatternIssue[] = [];+ for (const [field, patterns] of [+ ["includePaths", include],+ ["excludePaths", exclude],+ ] as const) {+ if (patterns.length > MAX_PATH_PATTERNS) {+ capIssues.push({+ kind: "field-cap",+ path: [field],+ message: `includePaths and excludePaths each accept at most ${MAX_PATH_PATTERNS} patterns.`,+ });+ }+ if (patterns.some(p => p.length > MAX_PATH_PATTERN_LENGTH)) {+ capIssues.push({+ kind: "field-cap",+ path: [field],+ message: `Each includePaths/excludePaths pattern must be at most ${MAX_PATH_PATTERN_LENGTH} characters.`,+ });+ }+ }+ if (capIssues.length > 0) return capIssues;++ const totalCount = include.length + exclude.length;+ if (totalCount > MAX_TOTAL_PATH_PATTERNS) {+ return [+ {+ kind: "budget",+ path: [],+ message: `includePaths and excludePaths together accept at most ${MAX_TOTAL_PATH_PATTERNS} patterns (got ${totalCount}).`,+ },+ ];+ }+ const totalChars = [...include, ...exclude].reduce(+ (sum, p) => sum + p.length,+ 0,+ );+ if (totalChars > MAX_TOTAL_PATH_PATTERN_CHARS) {+ return [+ {+ kind: "budget",+ path: [],+ message: `includePaths and excludePaths together accept at most ${MAX_TOTAL_PATH_PATTERN_CHARS} characters of patterns (got ${totalChars}).`,+ },+ ];+ }++ return [+ ...fieldRegexIssues(include, "includePaths"),+ ...fieldRegexIssues(exclude, "excludePaths"),+ ];+}++// Zod refinement wrapper around collectPathPatternIssues. Zod still runs this+// refinement when pathPatternsSchema has already reported a per-field count or+// length violation, so those are not reported a second time here.+export function addPathRegexIssues(+ fields: PathPatternFields,+ ctx: z.RefinementCtx,+): void {+ for (const issue of collectPathPatternIssues(fields)) {+ if (issue.kind === "field-cap") continue;+ ctx.addIssue({ code: "custom", path: issue.path, message: issue.message });+ }+}+// Link filtering compiles includePaths/excludePaths with the Rust `regex` crate// (RE2-style: no look-around or backreferences). Historically an unsupported// pattern compiled fine in most clients' regex flavor but was silently dropped// by the engine, so the paths it was meant to filter got crawled anyway. Reject// such patterns up front with a message that points at the actual limitation.-export function addPathRegexIssues(- patterns: string[] | undefined,- field: "includePaths" | "excludePaths",- ctx: z.RefinementCtx,-): void {- if (!patterns || patterns.length === 0) return;- // Zod still runs this refinement when pathPatternsSchema has already reported- // a count or length violation, so re-check the caps here: they are what bound- // the total compile work a request can demand, and an over-cap request has- // already been rejected.- if (- patterns.length > MAX_PATH_PATTERNS ||- patterns.some(p => p.length > MAX_PATH_PATTERN_LENGTH)- ) {- return;- }- for (const { pattern, error } of validateRegexes(patterns)) {+function fieldRegexIssues(+ patterns: string[],+ field: PathPatternField,+): PathPatternIssue[] {+ if (patterns.length === 0) return [];+ return validateRegexes(patterns).map(({ pattern, error }) => {const summary = summarizeRegexError(error);- ctx.addIssue({- code: "custom",+ return {+ kind: "syntax",path: [field],message:`Invalid ${field} pattern ${JSON.stringify(pattern)}: ${summary}. ` +`${field} patterns use Rust regex (RE2-style) syntax.${regexErrorHint(summary)}`,- });- }+ };+ });}function summarizeRegexError(error: string): string {