Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): add sitemap-only support (#2726)
firecrawl-spring[bot]7b7701129123
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 82111f9e5..2d4e439da 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -13,6 +13,7 @@ import {crawl,crawlOngoing,crawlStart,+ map,Identity,idmux,scrapeTimeout,@@ -23,6 +24,13 @@ import { describe, it, expect } from "@jest/globals";let identity: Identity;+const normalizeUrlForCompare = (value: string) => {+ const url = new URL(value);+ url.hash = "";+ const href = url.href;+ return href.endsWith("/") ? href.slice(0, -1) : href;+};+beforeAll(async () => {identity = await idmux({name: "crawl",@@ -69,6 +77,97 @@ describe("Crawl tests", () => {10 * scrapeTimeout,);+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "works with sitemap: only",+ async () => {+ const results = await crawl(+ {+ url: base,+ limit: 10,+ sitemap: "only",+ },+ identity,+ );++ expect(results.completed).toBeGreaterThan(0);+ },+ 10 * scrapeTimeout,+ );++ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(+ "sitemap-only results are subset of map-only + start URL",+ async () => {+ const mapResponse = await map(+ {+ url: base,+ sitemap: "only",+ includeSubdomains: false,+ ignoreQueryParameters: false,+ limit: 500,+ },+ identity,+ );++ expect(mapResponse.statusCode).toBe(200);+ expect(mapResponse.body.success).toBe(true);++ const sitemapUrls = new Set(+ mapResponse.body.links.map(link => normalizeUrlForCompare(link.url)),+ );+ const baseNormalized = normalizeUrlForCompare(base);++ const results = await crawl(+ {+ url: base,+ limit: 50,+ sitemap: "only",+ },+ identity,+ );++ expect(results.success).toBe(true);+ if (results.success) {+ for (const page of results.data) {+ const pageUrl =+ page.metadata.url ?? page.metadata.sourceURL ?? base;+ const normalized = normalizeUrlForCompare(pageUrl);+ expect(+ normalized === baseNormalized || sitemapUrls.has(normalized),+ ).toBe(true);+ }+ }+ },+ 10 * scrapeTimeout,+ );++ concurrentIf(TEST_PRODUCTION)(+ "no sitemap found -> start URL only",+ async () => {+ const noSitemapUrl = "https://example.com";+ const results = await crawl(+ {+ url: noSitemapUrl,+ limit: 10,+ sitemap: "only",+ },+ identity,+ );++ expect(results.success).toBe(true);+ if (results.success) {+ expect(results.data.length).toBe(1);+ const pageUrl =+ results.data[0].metadata.url ??+ results.data[0].metadata.sourceURL ??+ noSitemapUrl;+ expect(normalizeUrlForCompare(pageUrl)).toBe(+ normalizeUrlForCompare(noSitemapUrl),+ );+ }+ },+ 10 * scrapeTimeout,+ );+concurrentIf(ALLOW_TEST_SUITE_WEBSITE)("filters URLs properly",async () => {diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex cb1e98b8e..cdee18ca6 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -19,6 +19,7 @@ import {BatchScrapeRequestInput,SearchRequest,SearchRequestInput,+ toV2CrawlerOptions,} from "../../../controllers/v2/types";describe("V2 Types Validation", () => {@@ -606,11 +607,11 @@ describe("V2 Types Validation", () => {it("should handle sitemap enum values", () => {const input: CrawlRequestInput = {url: "https://example.com",- sitemap: "include",+ sitemap: "only",};const result = crawlRequestSchema.parse(input);- expect(result.sitemap).toBe("include");+ expect(result.sitemap).toBe("only");});it("should reject invalid sitemap value", () => {@@ -621,6 +622,15 @@ describe("V2 Types Validation", () => {expect(() => crawlRequestSchema.parse(input)).toThrow();});++ it("should map sitemapOnly to sitemap=only", () => {+ const result = toV2CrawlerOptions({+ sitemapOnly: true,+ ignoreSitemap: false,+ });++ expect(result.sitemap).toBe("only");+ });});describe("mapRequestSchema", () => {diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.tsindex cdeaf2b28..312a3af9a 100644--- a/apps/api/src/controllers/v2/crawl-params-preview.ts+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts@@ -29,7 +29,7 @@ type CrawlParamsPreviewResponse =crawlEntireDomain?: boolean;allowExternalLinks?: boolean;allowSubdomains?: boolean;- sitemap?: "skip" | "include";+ sitemap?: "skip" | "include" | "only";ignoreQueryParameters?: boolean;deduplicateSimilarURLs?: boolean;delay?: number;diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex f7599f786..4cb2b3beb 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -868,7 +868,7 @@ export const crawlerOptions = z.strictObject({allowExternalLinks: z.boolean().prefault(false),allowSubdomains: z.boolean().prefault(false),ignoreRobotsTxt: z.boolean().prefault(false),- sitemap: z.enum(["skip", "include"]).prefault("include"),+ sitemap: z.enum(["skip", "include", "only"]).prefault("include"),deduplicateSimilarURLs: z.boolean().prefault(true),ignoreQueryParameters: z.boolean().prefault(false),regexOnFullURL: z.boolean().prefault(false),@@ -1283,6 +1283,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,ignoreSitemap: x.sitemap === "skip",+ sitemapOnly: x.sitemap === "only",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,@@ -1301,7 +1302,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,- sitemap: x.ignoreSitemap ? "skip" : "include",+ sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,@@ -1326,7 +1327,7 @@ function fromV0CrawlerOptions(allowExternalLinks: x.allowExternalContentLinks,allowSubdomains: x.allowSubdomains,ignoreRobotsTxt: x.ignoreRobotsTxt,- sitemap: x.ignoreSitemap ? "skip" : "include",+ sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",deduplicateSimilarURLs: x.deduplicateSimilarURLs,ignoreQueryParameters: x.ignoreQueryParameters,regexOnFullURL: x.regexOnFullURL,diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 20aa1bbbd..c5e0f0356 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -356,78 +356,81 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,);- const links = await crawler.filterLinks(- await crawler.extractLinksFromHTML(- rawHtml ?? "",- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,- ),- Infinity,- sc.crawlerOptions?.maxDepth ?? 10,- );- logger.debug("Discovered " + links.links.length + " links...", {- linksLength: links.links.length,- });+ if (!sc.crawlerOptions?.sitemapOnly) {+ const links = await crawler.filterLinks(+ await crawler.extractLinksFromHTML(+ rawHtml ?? "",+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,+ ),+ Infinity,+ sc.crawlerOptions?.maxDepth ?? 10,+ );+ logger.debug("Discovered " + links.links.length + " links...", {+ linksLength: links.links.length,+ });- // Store robots blocked URLs in Redis set- for (const [url, reason] of links.denialReasons) {- if (reason === "URL blocked by robots.txt") {- await recordRobotsBlocked(job.data.crawl_id, url);+ // Store robots blocked URLs in Redis set+ for (const [url, reason] of links.denialReasons) {+ if (reason === "URL blocked by robots.txt") {+ await recordRobotsBlocked(job.data.crawl_id, url);+ }}- }- for (const link of links.links) {- if (await lockURL(job.data.crawl_id, sc, link)) {- // This seems to work really welel- const jobPriority = await getJobPriority({- team_id: sc.team_id,- basePriority: job.data.crawl_id ? 20 : 10,- });- const jobId = uuidv7();-- logger.debug(- "Determined job priority " +- jobPriority +- " for URL " +- JSON.stringify(link),- { jobPriority, url: link },- );-- await addScrapeJob(- {- url: link,- mode: "single_urls",+ for (const link of links.links) {+ if (await lockURL(job.data.crawl_id, sc, link)) {+ // This seems to work really welel+ const jobPriority = await getJobPriority({team_id: sc.team_id,- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),- internalOptions: sc.internalOptions,- crawlerOptions: {- ...sc.crawlerOptions,- currentDiscoveryDepth:- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,+ basePriority: job.data.crawl_id ? 20 : 10,+ });+ const jobId = uuidv7();++ logger.debug(+ "Determined job priority " ++ jobPriority ++ " for URL " ++ JSON.stringify(link),+ { jobPriority, url: link },+ );++ await addScrapeJob(+ {+ url: link,+ mode: "single_urls",+ team_id: sc.team_id,+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),+ internalOptions: sc.internalOptions,+ crawlerOptions: {+ ...sc.crawlerOptions,+ currentDiscoveryDepth:+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) ++ 1,+ },+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ requestId: job.data.requestId,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention: job.data.zeroDataRetention,+ apiKeyId: job.data.apiKeyId,},- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- requestId: job.data.requestId,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- apiKeyId: job.data.apiKeyId,- },- jobId,- jobPriority,- );-- await addCrawlJob(job.data.crawl_id, jobId, logger);- logger.debug("Added job for URL " + JSON.stringify(link), {- jobPriority,- url: link,- newJobId: jobId,- });- } else {- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!- // logger.debug("Could not lock URL " + JSON.stringify(link), {- // url: link,- // });+ jobId,+ jobPriority,+ );++ await addCrawlJob(job.data.crawl_id, jobId, logger);+ logger.debug("Added job for URL " + JSON.stringify(link), {+ jobPriority,+ url: link,+ newJobId: jobId,+ });+ } else {+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!+ // logger.debug("Could not lock URL " + JSON.stringify(link), {+ // url: link,+ // });+ }}}@@ -719,7 +722,7 @@ async function kickoffGetIndexLinks(crawler: WebCrawler,url: string,) {- if (sc.crawlerOptions.ignoreSitemap) {+ if (sc.crawlerOptions.ignoreSitemap || sc.crawlerOptions.sitemapOnly) {return [];}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 71e6d4e0a..02be36336 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -460,7 +460,7 @@ export interface CrawlOptions {excludePaths?: string[] | null;includePaths?: string[] | null;maxDiscoveryDepth?: number | null;- sitemap?: 'skip' | 'include';+ sitemap?: 'skip' | 'include' | 'only';ignoreQueryParameters?: boolean;limit?: number | null;crawlEntireDomain?: boolean;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 3ab4358b0..fcda4c526 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.13.1"+__version__ = "4.13.2"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.pyindex 039870fb8..81ad03521 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py@@ -42,7 +42,7 @@ class TestCrawlParamsData:assert data.include_paths is Noneassert data.exclude_paths is Noneassert data.max_discovery_depth is None- assert data.ignore_sitemap is False+ assert data.sitemap is Noneassert data.limit is Noneassert data.crawl_entire_domain is Falseassert data.allow_external_links is False@@ -67,4 +67,4 @@ class TestCrawlParamsData:assert data.limit == 50assert data.crawl_entire_domain is Trueassert data.allow_external_links is False- assert data.warning == "Test warning"\ No newline at end of file+ assert data.warning == "Test warning"diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 71e5b0bdd..c5df6d9d8 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -229,7 +229,8 @@ class FirecrawlClient:exclude_paths: Optional[List[str]] = None,include_paths: Optional[List[str]] = None,max_discovery_depth: Optional[int] = None,- ignore_sitemap: bool = False,+ sitemap: Optional[Literal["only", "include", "skip"]] = None,+ ignore_sitemap: Optional[bool] = None,ignore_query_parameters: bool = False,limit: Optional[int] = None,crawl_entire_domain: bool = False,@@ -254,7 +255,8 @@ class FirecrawlClient:exclude_paths: Patterns of URLs to excludeinclude_paths: Patterns of URLs to includemax_discovery_depth: Maximum depth for finding new URLs- ignore_sitemap: Skip sitemap.xml processing+ sitemap: Sitemap usage mode ("only" | "include" | "skip")+ ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)ignore_query_parameters: Ignore URL parameterslimit: Maximum pages to crawlcrawl_entire_domain: Follow parent directory links@@ -277,25 +279,32 @@ class FirecrawlClient:Exception: If the crawl fails to start or completeTimeoutError: If timeout is reached"""- request = CrawlRequest(- url=url,- prompt=prompt,- exclude_paths=exclude_paths,- include_paths=include_paths,- max_discovery_depth=max_discovery_depth,- ignore_sitemap=ignore_sitemap,- ignore_query_parameters=ignore_query_parameters,- limit=limit,- crawl_entire_domain=crawl_entire_domain,- allow_external_links=allow_external_links,- allow_subdomains=allow_subdomains,- delay=delay,- max_concurrency=max_concurrency,- webhook=webhook,- scrape_options=scrape_options,- zero_data_retention=zero_data_retention,- integration=integration,- )+ resolved_sitemap = sitemap+ if resolved_sitemap is None and ignore_sitemap is not None:+ resolved_sitemap = "skip" if ignore_sitemap else "include"++ request_kwargs = {+ "url": url,+ "prompt": prompt,+ "exclude_paths": exclude_paths,+ "include_paths": include_paths,+ "max_discovery_depth": max_discovery_depth,+ "ignore_query_parameters": ignore_query_parameters,+ "limit": limit,+ "crawl_entire_domain": crawl_entire_domain,+ "allow_external_links": allow_external_links,+ "allow_subdomains": allow_subdomains,+ "delay": delay,+ "max_concurrency": max_concurrency,+ "webhook": webhook,+ "scrape_options": scrape_options,+ "zero_data_retention": zero_data_retention,+ "integration": integration,+ }+ if resolved_sitemap is not None:+ request_kwargs["sitemap"] = resolved_sitemap++ request = CrawlRequest(**request_kwargs)return crawl_module.crawl(self.http_client,@@ -313,7 +322,8 @@ class FirecrawlClient:exclude_paths: Optional[List[str]] = None,include_paths: Optional[List[str]] = None,max_discovery_depth: Optional[int] = None,- ignore_sitemap: bool = False,+ sitemap: Optional[Literal["only", "include", "skip"]] = None,+ ignore_sitemap: Optional[bool] = None,ignore_query_parameters: bool = False,limit: Optional[int] = None,crawl_entire_domain: bool = False,@@ -335,7 +345,8 @@ class FirecrawlClient:exclude_paths: Patterns of URLs to excludeinclude_paths: Patterns of URLs to includemax_discovery_depth: Maximum depth for finding new URLs- ignore_sitemap: Skip sitemap.xml processing+ sitemap: Sitemap usage mode ("only" | "include" | "skip")+ ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)ignore_query_parameters: Ignore URL parameterslimit: Maximum pages to crawlcrawl_entire_domain: Follow parent directory links@@ -354,25 +365,32 @@ class FirecrawlClient:ValueError: If request is invalidException: If the crawl operation fails to start"""- request = CrawlRequest(- url=url,- prompt=prompt,- exclude_paths=exclude_paths,- include_paths=include_paths,- max_discovery_depth=max_discovery_depth,- ignore_sitemap=ignore_sitemap,- ignore_query_parameters=ignore_query_parameters,- limit=limit,- crawl_entire_domain=crawl_entire_domain,- allow_external_links=allow_external_links,- allow_subdomains=allow_subdomains,- delay=delay,- max_concurrency=max_concurrency,- webhook=webhook,- scrape_options=scrape_options,- zero_data_retention=zero_data_retention,- integration=integration,- )+ resolved_sitemap = sitemap+ if resolved_sitemap is None and ignore_sitemap is not None:+ resolved_sitemap = "skip" if ignore_sitemap else "include"++ request_kwargs = {+ "url": url,+ "prompt": prompt,+ "exclude_paths": exclude_paths,+ "include_paths": include_paths,+ "max_discovery_depth": max_discovery_depth,+ "ignore_query_parameters": ignore_query_parameters,+ "limit": limit,+ "crawl_entire_domain": crawl_entire_domain,+ "allow_external_links": allow_external_links,+ "allow_subdomains": allow_subdomains,+ "delay": delay,+ "max_concurrency": max_concurrency,+ "webhook": webhook,+ "scrape_options": scrape_options,+ "zero_data_retention": zero_data_retention,+ "integration": integration,+ }+ if resolved_sitemap is not None:+ request_kwargs["sitemap"] = resolved_sitemap++ request = CrawlRequest(**request_kwargs)return crawl_module.start_crawl(self.http_client, request)@@ -977,4 +995,4 @@ class FirecrawlClient:poll_interval=poll_interval,timeout=wait_timeout,)-\ No newline at end of file+diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex ee20f34e3..c2493421f 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -81,6 +81,13 @@ class AsyncFirecrawlClient:return await async_search.search(self.async_http_client, request)async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:+ sitemap = kwargs.pop("sitemap", None)+ ignore_sitemap = kwargs.pop("ignore_sitemap", None)+ if sitemap is None and ignore_sitemap is not None:+ sitemap = "skip" if ignore_sitemap else "include"+ if sitemap is not None:+ kwargs["sitemap"] = sitemap+request = CrawlRequest(url=url, **kwargs)return await async_crawl.start_crawl(self.async_http_client, request)@@ -414,4 +421,3 @@ class AsyncFirecrawlClient:timeout: Optional[int] = None,) -> AsyncWatcher:return AsyncWatcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)-diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.pyindex 5d1d78830..c2a10d8d7 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py@@ -43,7 +43,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:"include_paths": "includePaths","exclude_paths": "excludePaths","max_discovery_depth": "maxDiscoveryDepth",- "ignore_sitemap": "ignoreSitemap",+ "sitemap": "sitemap","ignore_query_parameters": "ignoreQueryParameters","crawl_entire_domain": "crawlEntireDomain","allow_external_links": "allowExternalLinks",@@ -272,7 +272,7 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ"includePaths": "include_paths","excludePaths": "exclude_paths","maxDiscoveryDepth": "max_discovery_depth",- "ignoreSitemap": "ignore_sitemap",+ "sitemap": "sitemap","ignoreQueryParameters": "ignore_query_parameters","crawlEntireDomain": "crawl_entire_domain","allowExternalLinks": "allow_external_links",@@ -348,4 +348,3 @@ async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:"options": c.get("options"),})return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])-diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 673870939..297332a04 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -572,7 +572,7 @@ class CrawlRequest(BaseModel):exclude_paths: Optional[List[str]] = Noneinclude_paths: Optional[List[str]] = Nonemax_discovery_depth: Optional[int] = None- sitemap: Literal["skip", "include"] = "include"+ sitemap: Literal["skip", "include", "only"] = "include"ignore_query_parameters: bool = Falselimit: Optional[int] = Nonecrawl_entire_domain: bool = False@@ -664,7 +664,7 @@ class CrawlParamsData(BaseModel):include_paths: Optional[List[str]] = Noneexclude_paths: Optional[List[str]] = Nonemax_discovery_depth: Optional[int] = None- ignore_sitemap: bool = False+ sitemap: Optional[Literal["skip", "include", "only"]] = Noneignore_query_parameters: bool = Falselimit: Optional[int] = Nonecrawl_entire_domain: bool = False