Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(v2/crawl): change sitemap param
Gergő Móricz548f19aafc04
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex 9e5295109..8eb7035f9 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -29,7 +29,7 @@ type CrawlParamsPreviewResponse = {crawlEntireDomain?: boolean;allowExternalLinks?: boolean;allowSubdomains?: boolean;- ignoreSitemap?: boolean;+ sitemap?: "skip" | "include";ignoreQueryParameters?: boolean;deduplicateSimilarURLs?: boolean;delay?: number;diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex e1ee585c0..7c58367ee 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -90,7 +90,6 @@ export async function getMapResults({url: string;search?: string;limit?: number;- ignoreSitemap?: boolean;includeSubdomains?: boolean;crawlerOptions?: any;teamId: string;@@ -129,7 +128,7 @@ export async function getMapResults({} catch (_) {}// If sitemapOnly is true, only get links from sitemap- if (crawlerOptions.sitemapOnly) {+ if (crawlerOptions.sitemap === "only") {const sitemap = await crawler.tryGetSitemap((urls) => {urls.forEach((x) => {@@ -210,7 +209,7 @@ export async function getMapResults({// If sitemap is not ignored, fetch sitemap// This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days- if (!crawlerOptions.ignoreSitemap) {+ if (crawlerOptions.sitemap === "include") {try {await crawler.tryGetSitemap((urls) => {@@ -322,11 +321,10 @@ export async function mapController(url: req.body.url,search: req.body.search,limit: req.body.limit,- ignoreSitemap: req.body.sitemap === "skip",includeSubdomains: req.body.includeSubdomains,crawlerOptions: {...req.body,- sitemapOnly: req.body.sitemap === "only",+ sitemap: req.body.sitemap,},origin: req.body.origin,teamId: req.auth.team_id,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex ffd02e9d5..489fa8cee 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -540,7 +540,7 @@ const crawlerOptions = zallowExternalLinks: z.boolean().default(false),allowSubdomains: z.boolean().default(false),ignoreRobotsTxt: z.boolean().default(false),- ignoreSitemap: z.boolean().default(false),+ sitemap: z.enum(["skip", "include"]).default("include"),deduplicateSimilarURLs: z.boolean().default(true),ignoreQueryParameters: z.boolean().default(false),regexOnFullURL: z.boolean().default(false),@@ -601,7 +601,7 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;export const mapRequestSchema = crawlerOptions- .omit({ ignoreSitemap: true })+ .omit({ sitemap: true }).extend({url,origin: z.string().optional().default("api"),@@ -991,7 +991,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {allowExternalContentLinks: x.allowExternalLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,- ignoreSitemap: x.ignoreSitemap,+ ignoreSitemap: x.sitemap === "skip",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,@@ -1011,7 +1011,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,- ignoreSitemap: x.ignoreSitemap,+ sitemap: x.ignoreSitemap ? "skip" : "include",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,@@ -1034,7 +1034,7 @@ export function fromV0CrawlerOptions(x: any, teamId: string): {allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,- ignoreSitemap: x.ignoreSitemap,+ sitemap: x.ignoreSitemap ? "skip" : "include",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.tsindex f0b938e70..53fbb1fe9 100644--- a/apps/api/src/lib/extract/build-prompts.ts+++ b/apps/api/src/lib/extract/build-prompts.ts@@ -138,7 +138,7 @@ Generate a JSON object with crawler configuration options. Available options:- allowExternalLinks: boolean - Whether to follow external links- allowSubdomains: boolean - Whether to crawl subdomains- ignoreRobotsTxt: boolean - Whether to ignore robots.txt-- ignoreSitemap: boolean - Whether to ignore sitemap+- sitemap: "skip" | "include" - Whether to ignore sitemap- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs- ignoreQueryParameters: boolean - Whether to ignore query parameters- regexOnFullURL: boolean - Whether regex patterns apply to full URLdiff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.tsindex 25ff35eb0..d9e80a7a1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts@@ -1140,7 +1140,7 @@ Available crawler options:- crawlEntireDomain: boolean - Allows the crawler to follow internal links to sibling or parent URLs, not just child paths. false: Only crawls deeper (child) URLs. → e.g. /features/feature-1 → /features/feature-1/tips ✅ → Won't follow /pricing or / ❌. true: Crawls any internal links, including siblings and parents. → e.g. /features/feature-1 → /pricing, /, etc. ✅. Use true for broader internal coverage beyond nested paths. Default: false- allowExternalLinks: boolean - Allows the crawler to follow links to external websites. Default: false- allowSubdomains: boolean - Allows the crawler to follow links to subdomains of the main domain. Default: false-- ignoreSitemap: boolean - Ignore the website sitemap when crawling. Default: false+- sitemap: "skip" | "include" - Whether to ignore sitemap. Default: "include"- ignoreQueryParameters: boolean - Do not re-scrape the same path with different (or none) query parameters. Default: false- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs- delay: number - Delay in seconds between scrapes. This helps respect website rate limits.diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.tsindex 6937f7b5f..0ba2a485f 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts@@ -146,7 +146,7 @@ describe("v2.crawl e2e", () => {includePaths: ["/blog/*", "/docs/*"],excludePaths: ["/admin/*"],maxDiscoveryDepth: 3,- ignoreSitemap: false,+ sitemap: "skip",ignoreQueryParameters: true,limit: 5,crawlEntireDomain: true,diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.tsindex c8df2ebb5..4d9ba3383 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts@@ -17,7 +17,7 @@ export interface CrawlRequest {excludePaths?: string[] | null;includePaths?: string[] | null;maxDiscoveryDepth?: number | null;- ignoreSitemap?: boolean;+ sitemap?: "skip" | "include";ignoreQueryParameters?: boolean;limit?: number | null;crawlEntireDomain?: boolean;@@ -37,7 +37,7 @@ function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {if (request.excludePaths) data.excludePaths = request.excludePaths;if (request.includePaths) data.includePaths = request.includePaths;if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;- if (request.ignoreSitemap != null) data.ignoreSitemap = request.ignoreSitemap;+ if (request.sitemap != null) data.sitemap = request.sitemap;if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;if (request.limit != null) data.limit = request.limit;if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;