Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): add maxDiscoveryDepth (#1329)
Gergő Móricz7cf2e52fe6d4
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex f388243d1..599e34cfc 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -53,4 +53,21 @@ describe("Crawl tests", () => {}}}, 120000);++ it.concurrent("discovers URLs properly when maxDiscoveryDepth is provided", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ ignoreSitemap: true,+ maxDiscoveryDepth: 1,+ limit: 10,+ });++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.data.length).toBeGreaterThan(1);+ for (const page of res.data) {+ expect(page.metadata.url ?? page.metadata.sourceURL).not.toMatch(/^https:\/\/(www\.)?firecrawl\.dev\/blog\/.+$/);+ }+ }+ }, 120000);});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 4f946f03a..5a71da9c6 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -440,6 +440,7 @@ const crawlerOptions = zincludePaths: z.string().array().default([]),excludePaths: z.string().array().default([]),maxDepth: z.number().default(10), // default?+ maxDiscoveryDepth: z.number().optional(),limit: z.number().default(10000), // default?allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME???allowExternalLinks: z.boolean().default(false),@@ -793,6 +794,8 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,+ maxDiscoveryDepth: x.maxDiscoveryDepth,+ currentDiscoveryDepth: 0,};}@@ -814,7 +817,8 @@ export function fromLegacyCrawlerOptions(x: any): {deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,- }),+ maxDiscoveryDepth: x.maxDiscoveryDepth,+ }),internalOptions: {v0CrawlOnlyUrls: x.returnOnlyUrls,},diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 256d74352..b31605c70 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -379,6 +379,7 @@ export function crawlToCrawler(id: string,sc: StoredCrawl,newBase?: string,+ crawlerOptions?: any,): WebCrawler {const crawler = new WebCrawler({jobId: id,@@ -399,6 +400,8 @@ export function crawlToCrawler(allowSubdomains: sc.crawlerOptions?.allowSubdomains ?? false,ignoreRobotsTxt: sc.crawlerOptions?.ignoreRobotsTxt ?? false,regexOnFullURL: sc.crawlerOptions?.regexOnFullURL ?? false,+ maxDiscoveryDepth: sc.crawlerOptions?.maxDiscoveryDepth,+ currentDiscoveryDepth: crawlerOptions?.currentDiscoveryDepth ?? 0,});if (sc.robots !== undefined) {diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex ea93110ad..5f26f817c 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -31,6 +31,8 @@ export class WebCrawler {private regexOnFullURL: boolean;private logger: typeof _logger;private sitemapsHit: Set<string> = new Set();+ private maxDiscoveryDepth: number | undefined;+ private currentDiscoveryDepth: number;constructor({jobId,@@ -47,6 +49,8 @@ export class WebCrawler {allowSubdomains = false,ignoreRobotsTxt = false,regexOnFullURL = false,+ maxDiscoveryDepth,+ currentDiscoveryDepth,}: {jobId: string;initialUrl: string;@@ -62,6 +66,8 @@ export class WebCrawler {allowSubdomains?: boolean;ignoreRobotsTxt?: boolean;regexOnFullURL?: boolean;+ maxDiscoveryDepth?: number;+ currentDiscoveryDepth?: number;}) {this.jobId = jobId;this.initialUrl = initialUrl;@@ -81,6 +87,8 @@ export class WebCrawler {this.ignoreRobotsTxt = ignoreRobotsTxt ?? false;this.regexOnFullURL = regexOnFullURL ?? false;this.logger = _logger.child({ crawlId: this.jobId, module: "WebCrawler" });+ this.maxDiscoveryDepth = maxDiscoveryDepth;+ this.currentDiscoveryDepth = currentDiscoveryDepth ?? 0;}public filterLinks(@@ -89,6 +97,11 @@ export class WebCrawler {maxDepth: number,fromMap: boolean = false,): string[] {+ if (this.currentDiscoveryDepth === this.maxDiscoveryDepth) {+ this.logger.debug("Max discovery depth hit, filtering off all links", { currentDiscoveryDepth: this.currentDiscoveryDepth, maxDiscoveryDepth: this.maxDiscoveryDepth });+ return [];+ }+// If the initial URL is a sitemap.xml, skip filteringif (this.initialUrl.endsWith("sitemap.xml") && fromMap) {return sitemapLinks.slice(0, limit);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 6287cf95e..5493da734 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -1044,6 +1044,7 @@ async function processJob(job: Job & { id: string }, token: string) {job.data.crawl_id,sc,doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,+ job.data.crawlerOptions,);const links = crawler.filterLinks(@@ -1088,6 +1089,10 @@ async function processJob(job: Job & { id: string }, token: string) {team_id: sc.team_id,scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),internalOptions: sc.internalOptions,+ crawlerOptions: {+ ...sc.crawlerOptions,+ currentDiscoveryDepth: (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,+ },plan: job.data.plan,origin: job.data.origin,crawl_id: job.data.crawl_id,diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 99e008d3f..ab09432e5 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -160,6 +160,7 @@ export interface CrawlParams {includePaths?: string[];excludePaths?: string[];maxDepth?: number;+ maxDiscoveryDepth?: number;limit?: number;allowBackwardLinks?: boolean;allowExternalLinks?: boolean;