Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: add followInternalLinks parameter as semantic replacement for allowBackwardLinks (#1684)
devin-ai-integration[bot]09aabbedb58b
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex 3e77b2cc7..8903458a3 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -100,4 +100,44 @@ describe("Crawl tests", () => {// }// }// }, 300000);++ it.concurrent("crawlEntireDomain parameter works", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ crawlEntireDomain: true,+ limit: 5,+ });++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ }+ }, 120000);++ it.concurrent("crawlEntireDomain takes precedence over allowBackwardLinks", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ allowBackwardLinks: false,+ crawlEntireDomain: true,+ limit: 5,+ });++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ }+ }, 120000);++ it.concurrent("backward compatibility - allowBackwardLinks still works", async () => {+ const res = await crawl({+ url: "https://firecrawl.dev",+ allowBackwardLinks: true,+ limit: 5,+ });++ expect(res.success).toBe(true);+ if (res.success) {+ expect(res.completed).toBeGreaterThan(0);+ }+ }, 120000);});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 42e40d051..ca35ab5bc 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -615,7 +615,8 @@ const crawlerOptions = zmaxDepth: z.number().default(10), // default?maxDiscoveryDepth: z.number().optional(),limit: z.number().default(10000), // default?- allowBackwardLinks: z.boolean().default(false), // >> TODO: CHANGE THIS NAME???+ allowBackwardLinks: z.boolean().default(false), // DEPRECATED: use crawlEntireDomain+ crawlEntireDomain: z.boolean().optional(),allowExternalLinks: z.boolean().default(false),allowSubdomains: z.boolean().default(false),ignoreRobotsTxt: z.boolean().default(false),@@ -632,7 +633,8 @@ const crawlerOptions = z// excludePaths?: string[];// maxDepth?: number;// limit?: number;-// allowBackwardLinks?: boolean; // >> TODO: CHANGE THIS NAME???+// allowBackwardLinks?: boolean; // DEPRECATED: use crawlEntireDomain+// crawlEntireDomain?: boolean;// allowExternalLinks?: boolean;// ignoreSitemap?: boolean;// };@@ -652,10 +654,15 @@ export const crawlRequestSchema = crawlerOptions.strict(strictMessage).refine((x) => extractRefine(x.scrapeOptions), extractRefineOpts).refine((x) => fire1Refine(x.scrapeOptions), fire1RefineOpts)- .transform((x) => ({- ...x,- scrapeOptions: extractTransform(x.scrapeOptions),- }));+ .transform((x) => {+ if (x.crawlEntireDomain !== undefined) {+ x.allowBackwardLinks = x.crawlEntireDomain;+ }+ return {+ ...x,+ scrapeOptions: extractTransform(x.scrapeOptions),+ };+ });// export type CrawlRequest = {// url: string;@@ -1041,7 +1048,7 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {maxDepth: x.maxDepth,limit: x.limit,generateImgAltText: false,- allowBackwardCrawling: x.allowBackwardLinks,+ allowBackwardCrawling: x.crawlEntireDomain ?? x.allowBackwardLinks,allowExternalContentLinks: x.allowExternalLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,@@ -1062,6 +1069,7 @@ export function toNewCrawlerOptions(x: any): CrawlerOptions {limit: x.limit,maxDepth: x.maxDepth,allowBackwardLinks: x.allowBackwardCrawling,+ crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,@@ -1085,6 +1093,7 @@ export function fromLegacyCrawlerOptions(x: any, teamId: string): {limit: x.maxCrawledLinks ?? x.limit,maxDepth: x.maxDepth,allowBackwardLinks: x.allowBackwardCrawling,+ crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,diff --git a/apps/js-sdk/firecrawl/src/index.ts b/apps/js-sdk/firecrawl/src/index.tsindex 05939ae01..5ec70baaf 100644--- a/apps/js-sdk/firecrawl/src/index.ts+++ b/apps/js-sdk/firecrawl/src/index.ts@@ -209,6 +209,7 @@ export interface CrawlParams {maxDiscoveryDepth?: number;limit?: number;allowBackwardLinks?: boolean;+ crawlEntireDomain?: boolean;allowExternalLinks?: boolean;ignoreSitemap?: boolean;scrapeOptions?: CrawlScrapeOptions;diff --git a/apps/python-sdk/firecrawl/firecrawl.py b/apps/python-sdk/firecrawl/firecrawl.pyindex bc1d0479c..b425cff53 100644--- a/apps/python-sdk/firecrawl/firecrawl.py+++ b/apps/python-sdk/firecrawl/firecrawl.py@@ -687,6 +687,7 @@ class FirecrawlApp:max_discovery_depth: Optional[int] = None,limit: Optional[int] = None,allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,scrape_options: Optional[ScrapeOptions] = None,@@ -710,7 +711,8 @@ class FirecrawlApp:max_depth (Optional[int]): Maximum crawl depthmax_discovery_depth (Optional[int]): Maximum depth for finding new URLslimit (Optional[int]): Maximum pages to crawl- allow_backward_links (Optional[bool]): Follow parent directory links+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processingscrape_options (Optional[ScrapeOptions]): Page scraping configuration@@ -749,7 +751,9 @@ class FirecrawlApp:crawl_params['maxDiscoveryDepth'] = max_discovery_depthif limit is not None:crawl_params['limit'] = limit- if allow_backward_links is not None:+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:crawl_params['allowBackwardLinks'] = allow_backward_linksif allow_external_links is not None:crawl_params['allowExternalLinks'] = allow_external_links@@ -802,6 +806,7 @@ class FirecrawlApp:max_discovery_depth: Optional[int] = None,limit: Optional[int] = None,allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,scrape_options: Optional[ScrapeOptions] = None,@@ -823,7 +828,8 @@ class FirecrawlApp:max_depth (Optional[int]): Maximum crawl depthmax_discovery_depth (Optional[int]): Maximum depth for finding new URLslimit (Optional[int]): Maximum pages to crawl- allow_backward_links (Optional[bool]): Follow parent directory links+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processingscrape_options (Optional[ScrapeOptions]): Page scraping configuration@@ -862,7 +868,9 @@ class FirecrawlApp:crawl_params['maxDiscoveryDepth'] = max_discovery_depthif limit is not None:crawl_params['limit'] = limit- if allow_backward_links is not None:+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:crawl_params['allowBackwardLinks'] = allow_backward_linksif allow_external_links is not None:crawl_params['allowExternalLinks'] = allow_external_links@@ -1051,6 +1059,7 @@ class FirecrawlApp:max_discovery_depth: Optional[int] = None,limit: Optional[int] = None,allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,scrape_options: Optional[ScrapeOptions] = None,@@ -1073,7 +1082,8 @@ class FirecrawlApp:max_depth (Optional[int]): Maximum crawl depthmax_discovery_depth (Optional[int]): Maximum depth for finding new URLslimit (Optional[int]): Maximum pages to crawl- allow_backward_links (Optional[bool]): Follow parent directory links+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processingscrape_options (Optional[ScrapeOptions]): Page scraping configuration@@ -2811,7 +2821,8 @@ class AsyncFirecrawlApp(FirecrawlApp):* limit - Maximum pages to crawlLink Following:- * allowBackwardLinks - Follow parent directory links+ * allowBackwardLinks - DEPRECATED: Use crawlEntireDomain instead+ * crawlEntireDomain - Follow parent directory links* allowExternalLinks - Follow external domain links* ignoreSitemap - Skip sitemap.xml processing@@ -3290,6 +3301,7 @@ class AsyncFirecrawlApp(FirecrawlApp):max_discovery_depth: Optional[int] = None,limit: Optional[int] = None,allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,scrape_options: Optional[ScrapeOptions] = None,@@ -3312,7 +3324,8 @@ class AsyncFirecrawlApp(FirecrawlApp):max_depth (Optional[int]): Maximum crawl depthmax_discovery_depth (Optional[int]): Maximum depth for finding new URLslimit (Optional[int]): Maximum pages to crawl- allow_backward_links (Optional[bool]): Follow parent directory links+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processingscrape_options (Optional[ScrapeOptions]): Page scraping configuration@@ -3350,7 +3363,9 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['maxDiscoveryDepth'] = max_discovery_depthif limit is not None:crawl_params['limit'] = limit- if allow_backward_links is not None:+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:crawl_params['allowBackwardLinks'] = allow_backward_linksif allow_external_links is not None:crawl_params['allowExternalLinks'] = allow_external_links@@ -3402,6 +3417,7 @@ class AsyncFirecrawlApp(FirecrawlApp):max_discovery_depth: Optional[int] = None,limit: Optional[int] = None,allow_backward_links: Optional[bool] = None,+ crawl_entire_domain: Optional[bool] = None,allow_external_links: Optional[bool] = None,ignore_sitemap: Optional[bool] = None,scrape_options: Optional[ScrapeOptions] = None,@@ -3424,7 +3440,8 @@ class AsyncFirecrawlApp(FirecrawlApp):max_depth (Optional[int]): Maximum crawl depthmax_discovery_depth (Optional[int]): Maximum depth for finding new URLslimit (Optional[int]): Maximum pages to crawl- allow_backward_links (Optional[bool]): Follow parent directory links+ allow_backward_links (Optional[bool]): DEPRECATED: Use crawl_entire_domain instead+ crawl_entire_domain (Optional[bool]): Follow parent directory linksallow_external_links (Optional[bool]): Follow external domain linksignore_sitemap (Optional[bool]): Skip sitemap.xml processingscrape_options (Optional[ScrapeOptions]): Page scraping configuration@@ -3458,7 +3475,9 @@ class AsyncFirecrawlApp(FirecrawlApp):crawl_params['maxDiscoveryDepth'] = max_discovery_depthif limit is not None:crawl_params['limit'] = limit- if allow_backward_links is not None:+ if crawl_entire_domain is not None:+ crawl_params['crawlEntireDomain'] = crawl_entire_domain+ elif allow_backward_links is not None:crawl_params['allowBackwardLinks'] = allow_backward_linksif allow_external_links is not None:crawl_params['allowExternalLinks'] = allow_external_links