Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(v2/crawl): change sitemap param

Gergő Móricz548f19aafc04
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index 9e5295109..8eb7035f9 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -29,7 +29,7 @@ type CrawlParamsPreviewResponse = {
crawlEntireDomain?: boolean;
allowExternalLinks?: boolean;
allowSubdomains?: boolean;
- ignoreSitemap?: boolean;
+ sitemap?: "skip" | "include";
ignoreQueryParameters?: boolean;
deduplicateSimilarURLs?: boolean;
delay?: number;
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index e1ee585c0..7c58367ee 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -90,7 +90,6 @@ export async function getMapResults({
url: string;
search?: string;
limit?: number;
- ignoreSitemap?: boolean;
includeSubdomains?: boolean;
crawlerOptions?: any;
teamId: string;
@@ -129,7 +128,7 @@ export async function getMapResults({
} catch (_) {}
// If sitemapOnly is true, only get links from sitemap
- if (crawlerOptions.sitemapOnly) {
+ if (crawlerOptions.sitemap === "only") {
const sitemap = await crawler.tryGetSitemap(
(urls) => {
urls.forEach((x) => {
@@ -210,7 +209,7 @@ export async function getMapResults({
// If sitemap is not ignored, fetch sitemap
// This will attempt to find it in the index at first, or fetch a fresh one if it's older than 2 days
- if (!crawlerOptions.ignoreSitemap) {
+ if (crawlerOptions.sitemap === "include") {
try {
await crawler.tryGetSitemap(
(urls) => {
@@ -322,11 +321,10 @@ export async function mapController(
url: req.body.url,
search: req.body.search,
limit: req.body.limit,
- ignoreSitemap: req.body.sitemap === "skip",
includeSubdomains: req.body.includeSubdomains,
crawlerOptions: {
...req.body,
- sitemapOnly: req.body.sitemap === "only",
+ sitemap: req.body.sitemap,
},
origin: req.body.origin,
teamId: req.auth.team_id,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index ffd02e9d5..489fa8cee 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -540,7 +540,7 @@ const crawlerOptions = z
allowExternalLinks: z.boolean().default(false),
allowSubdomains: z.boolean().default(false),
ignoreRobotsTxt: z.boolean().default(false),
- ignoreSitemap: z.boolean().default(false),
+ sitemap: z.enum(["skip", "include"]).default("include"),
deduplicateSimilarURLs: z.boolean().default(true),
ignoreQueryParameters: z.boolean().default(false),
regexOnFullURL: z.boolean().default(false),
@@ -601,7 +601,7 @@ export type CrawlRequest = z.infer<typeof crawlRequestSchema>;
export type CrawlRequestInput = z.input<typeof crawlRequestSchema>;
export const mapRequestSchema = crawlerOptions
- .omit({ ignoreSitemap: true })
+ .omit({ sitemap: true })
.extend({
url,
origin: z.string().optional().default("api"),
@@ -991,7 +991,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {
allowExternalContentLinks: x.allowExternalLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
- ignoreSitemap: x.ignoreSitemap,
+ ignoreSitemap: x.sitemap === "skip",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
@@ -1011,7 +1011,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {
allowExternalLinks: x.allowExternalContentLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
- ignoreSitemap: x.ignoreSitemap,
+ sitemap: x.ignoreSitemap ? "skip" : "include",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
@@ -1034,7 +1034,7 @@ export function fromV0CrawlerOptions(x: any, teamId: string): {
allowExternalLinks: x.allowExternalContentLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
- ignoreSitemap: x.ignoreSitemap,
+ sitemap: x.ignoreSitemap ? "skip" : "include",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
diff --git a/apps/api/src/lib/extract/build-prompts.ts b/apps/api/src/lib/extract/build-prompts.ts
index f0b938e70..53fbb1fe9 100644
--- a/apps/api/src/lib/extract/build-prompts.ts
+++ b/apps/api/src/lib/extract/build-prompts.ts
@@ -138,7 +138,7 @@ Generate a JSON object with crawler configuration options. Available options:
- allowExternalLinks: boolean - Whether to follow external links
- allowSubdomains: boolean - Whether to crawl subdomains
- ignoreRobotsTxt: boolean - Whether to ignore robots.txt
-- ignoreSitemap: boolean - Whether to ignore sitemap
+- sitemap: "skip" | "include" - Whether to ignore sitemap
- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs
- ignoreQueryParameters: boolean - Whether to ignore query parameters
- regexOnFullURL: boolean - Whether regex patterns apply to full URL
diff --git a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
index 25ff35eb0..d9e80a7a1 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/llmExtract.ts
@@ -1140,7 +1140,7 @@ Available crawler options:
- crawlEntireDomain: boolean - Allows the crawler to follow internal links to sibling or parent URLs, not just child paths. false: Only crawls deeper (child) URLs. → e.g. /features/feature-1 → /features/feature-1/tips ✅ → Won't follow /pricing or / ❌. true: Crawls any internal links, including siblings and parents. → e.g. /features/feature-1 → /pricing, /, etc. ✅. Use true for broader internal coverage beyond nested paths. Default: false
- allowExternalLinks: boolean - Allows the crawler to follow links to external websites. Default: false
- allowSubdomains: boolean - Allows the crawler to follow links to subdomains of the main domain. Default: false
-- ignoreSitemap: boolean - Ignore the website sitemap when crawling. Default: false
+- sitemap: "skip" | "include" - Whether to ignore sitemap. Default: "include"
- ignoreQueryParameters: boolean - Do not re-scrape the same path with different (or none) query parameters. Default: false
- deduplicateSimilarURLs: boolean - Whether to deduplicate similar URLs
- delay: number - Delay in seconds between scrapes. This helps respect website rate limits.
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
index 6937f7b5f..0ba2a485f 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts
@@ -146,7 +146,7 @@ describe("v2.crawl e2e", () => {
includePaths: ["/blog/*", "/docs/*"],
excludePaths: ["/admin/*"],
maxDiscoveryDepth: 3,
- ignoreSitemap: false,
+ sitemap: "skip",
ignoreQueryParameters: true,
limit: 5,
crawlEntireDomain: true,
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
index c8df2ebb5..4d9ba3383 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/crawl.ts
@@ -17,7 +17,7 @@ export interface CrawlRequest {
excludePaths?: string[] | null;
includePaths?: string[] | null;
maxDiscoveryDepth?: number | null;
- ignoreSitemap?: boolean;
+ sitemap?: "skip" | "include";
ignoreQueryParameters?: boolean;
limit?: number | null;
crawlEntireDomain?: boolean;
@@ -37,7 +37,7 @@ function prepareCrawlPayload(request: CrawlRequest): Record<string, unknown> {
if (request.excludePaths) data.excludePaths = request.excludePaths;
if (request.includePaths) data.includePaths = request.includePaths;
if (request.maxDiscoveryDepth != null) data.maxDiscoveryDepth = request.maxDiscoveryDepth;
- if (request.ignoreSitemap != null) data.ignoreSitemap = request.ignoreSitemap;
+ if (request.sitemap != null) data.sitemap = request.sitemap;
if (request.ignoreQueryParameters != null) data.ignoreQueryParameters = request.ignoreQueryParameters;
if (request.limit != null) data.limit = request.limit;
if (request.crawlEntireDomain != null) data.crawlEntireDomain = request.crawlEntireDomain;