Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
crawl: no more allowBackwardLinks
Gergő Móriczfd26d17b2a73
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 1bd97cb0a..2e2d9b182 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -155,33 +155,6 @@ describe("Crawl tests", () => {}}, 5 * scrapeTimeout);- it.concurrent("crawlEntireDomain takes precedence over allowBackwardLinks", async () => {- const res = await crawl({- url: "https://firecrawl.dev",- allowBackwardLinks: false,- crawlEntireDomain: true,- limit: 5,- }, identity);-- expect(res.success).toBe(true);- if (res.success) {- expect(res.completed).toBeGreaterThan(0);- }- }, 5 * scrapeTimeout);-- it.concurrent("backward compatibility - allowBackwardLinks still works", async () => {- const res = await crawl({- url: "https://firecrawl.dev",- allowBackwardLinks: true,- limit: 5,- }, identity);-- expect(res.success).toBe(true);- if (res.success) {- expect(res.completed).toBeGreaterThan(0);- }- }, 5 * scrapeTimeout);-it.concurrent("allowSubdomains parameter works", async () => {const res = await crawl({url: "https://firecrawl.dev",diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 825fc94a1..ade1edced 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -535,7 +535,6 @@ const crawlerOptions = zexcludePaths: z.string().array().default([]),maxDiscoveryDepth: z.number().optional(),limit: z.number().default(10000), // default?- allowBackwardLinks: z.boolean().default(false), // DEPRECATED: use crawlEntireDomaincrawlEntireDomain: z.boolean().optional(),allowExternalLinks: z.boolean().default(false),allowSubdomains: z.boolean().default(false),@@ -553,7 +552,6 @@ const crawlerOptions = z// excludePaths?: string[];// maxDepth?: number;// limit?: number;-// allowBackwardLinks?: boolean; // DEPRECATED: use crawlEntireDomain// crawlEntireDomain?: boolean;// allowExternalLinks?: boolean;// ignoreSitemap?: boolean;@@ -576,9 +574,6 @@ export const crawlRequestSchema = crawlerOptions.strict(strictMessage).refine((x) => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform((x) => {- if (x.crawlEntireDomain !== undefined) {- x.allowBackwardLinks = x.crawlEntireDomain;- }return {...x,scrapeOptions: extractTransform(x.scrapeOptions),@@ -987,7 +982,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {maxDepth: 9999,limit: x.limit,generateImgAltText: false,- allowBackwardCrawling: x.crawlEntireDomain ?? x.allowBackwardLinks,+ allowBackwardCrawling: x.crawlEntireDomain,allowExternalContentLinks: x.allowExternalLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,@@ -1006,7 +1001,6 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {includePaths: x.includes,excludePaths: x.excludes,limit: x.limit,- allowBackwardLinks: x.allowBackwardCrawling,crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,@@ -1029,7 +1023,6 @@ export function fromV0CrawlerOptions(x: any, teamId: string): {includePaths: x.includes,excludePaths: x.excludes,limit: x.maxCrawledLinks ?? x.limit,- allowBackwardLinks: x.allowBackwardCrawling,crawlEntireDomain: x.allowBackwardCrawling,allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,