Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(crawl): add sitemap-only support (#2726)

firecrawl-spring[bot]7b7701129123
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 82111f9e5..2d4e439da 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -13,6 +13,7 @@ import {
crawl,
crawlOngoing,
crawlStart,
+ map,
Identity,
idmux,
scrapeTimeout,
@@ -23,6 +24,13 @@ import { describe, it, expect } from "@jest/globals";
let identity: Identity;
+const normalizeUrlForCompare = (value: string) => {
+ const url = new URL(value);
+ url.hash = "";
+ const href = url.href;
+ return href.endsWith("/") ? href.slice(0, -1) : href;
+};
+
beforeAll(async () => {
identity = await idmux({
name: "crawl",
@@ -69,6 +77,97 @@ describe("Crawl tests", () => {
10 * scrapeTimeout,
);
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "works with sitemap: only",
+ async () => {
+ const results = await crawl(
+ {
+ url: base,
+ limit: 10,
+ sitemap: "only",
+ },
+ identity,
+ );
+
+ expect(results.completed).toBeGreaterThan(0);
+ },
+ 10 * scrapeTimeout,
+ );
+
+ concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
+ "sitemap-only results are subset of map-only + start URL",
+ async () => {
+ const mapResponse = await map(
+ {
+ url: base,
+ sitemap: "only",
+ includeSubdomains: false,
+ ignoreQueryParameters: false,
+ limit: 500,
+ },
+ identity,
+ );
+
+ expect(mapResponse.statusCode).toBe(200);
+ expect(mapResponse.body.success).toBe(true);
+
+ const sitemapUrls = new Set(
+ mapResponse.body.links.map(link => normalizeUrlForCompare(link.url)),
+ );
+ const baseNormalized = normalizeUrlForCompare(base);
+
+ const results = await crawl(
+ {
+ url: base,
+ limit: 50,
+ sitemap: "only",
+ },
+ identity,
+ );
+
+ expect(results.success).toBe(true);
+ if (results.success) {
+ for (const page of results.data) {
+ const pageUrl =
+ page.metadata.url ?? page.metadata.sourceURL ?? base;
+ const normalized = normalizeUrlForCompare(pageUrl);
+ expect(
+ normalized === baseNormalized || sitemapUrls.has(normalized),
+ ).toBe(true);
+ }
+ }
+ },
+ 10 * scrapeTimeout,
+ );
+
+ concurrentIf(TEST_PRODUCTION)(
+ "no sitemap found -> start URL only",
+ async () => {
+ const noSitemapUrl = "https://example.com";
+ const results = await crawl(
+ {
+ url: noSitemapUrl,
+ limit: 10,
+ sitemap: "only",
+ },
+ identity,
+ );
+
+ expect(results.success).toBe(true);
+ if (results.success) {
+ expect(results.data.length).toBe(1);
+ const pageUrl =
+ results.data[0].metadata.url ??
+ results.data[0].metadata.sourceURL ??
+ noSitemapUrl;
+ expect(normalizeUrlForCompare(pageUrl)).toBe(
+ normalizeUrlForCompare(noSitemapUrl),
+ );
+ }
+ },
+ 10 * scrapeTimeout,
+ );
+
concurrentIf(ALLOW_TEST_SUITE_WEBSITE)(
"filters URLs properly",
async () => {
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index cb1e98b8e..cdee18ca6 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -19,6 +19,7 @@ import {
BatchScrapeRequestInput,
SearchRequest,
SearchRequestInput,
+ toV2CrawlerOptions,
} from "../../../controllers/v2/types";
describe("V2 Types Validation", () => {
@@ -606,11 +607,11 @@ describe("V2 Types Validation", () => {
it("should handle sitemap enum values", () => {
const input: CrawlRequestInput = {
url: "https://example.com",
- sitemap: "include",
+ sitemap: "only",
};
const result = crawlRequestSchema.parse(input);
- expect(result.sitemap).toBe("include");
+ expect(result.sitemap).toBe("only");
});
it("should reject invalid sitemap value", () => {
@@ -621,6 +622,15 @@ describe("V2 Types Validation", () => {
expect(() => crawlRequestSchema.parse(input)).toThrow();
});
+
+ it("should map sitemapOnly to sitemap=only", () => {
+ const result = toV2CrawlerOptions({
+ sitemapOnly: true,
+ ignoreSitemap: false,
+ });
+
+ expect(result.sitemap).toBe("only");
+ });
});
describe("mapRequestSchema", () => {
diff --git a/apps/api/src/controllers/v2/crawl-params-preview.ts b/apps/api/src/controllers/v2/crawl-params-preview.ts
index cdeaf2b28..312a3af9a 100644
--- a/apps/api/src/controllers/v2/crawl-params-preview.ts
+++ b/apps/api/src/controllers/v2/crawl-params-preview.ts
@@ -29,7 +29,7 @@ type CrawlParamsPreviewResponse =
crawlEntireDomain?: boolean;
allowExternalLinks?: boolean;
allowSubdomains?: boolean;
- sitemap?: "skip" | "include";
+ sitemap?: "skip" | "include" | "only";
ignoreQueryParameters?: boolean;
deduplicateSimilarURLs?: boolean;
delay?: number;
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index f7599f786..4cb2b3beb 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -868,7 +868,7 @@ export const crawlerOptions = z.strictObject({
allowExternalLinks: z.boolean().prefault(false),
allowSubdomains: z.boolean().prefault(false),
ignoreRobotsTxt: z.boolean().prefault(false),
- sitemap: z.enum(["skip", "include"]).prefault("include"),
+ sitemap: z.enum(["skip", "include", "only"]).prefault("include"),
deduplicateSimilarURLs: z.boolean().prefault(true),
ignoreQueryParameters: z.boolean().prefault(false),
regexOnFullURL: z.boolean().prefault(false),
@@ -1283,6 +1283,7 @@ export function toV0CrawlerOptions(x: CrawlerOptions) {
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
ignoreSitemap: x.sitemap === "skip",
+ sitemapOnly: x.sitemap === "only",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
@@ -1301,7 +1302,7 @@ export function toV2CrawlerOptions(x: any): CrawlerOptions {
allowExternalLinks: x.allowExternalContentLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
- sitemap: x.ignoreSitemap ? "skip" : "include",
+ sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
@@ -1326,7 +1327,7 @@ function fromV0CrawlerOptions(
allowExternalLinks: x.allowExternalContentLinks,
allowSubdomains: x.allowSubdomains,
ignoreRobotsTxt: x.ignoreRobotsTxt,
- sitemap: x.ignoreSitemap ? "skip" : "include",
+ sitemap: x.sitemapOnly ? "only" : x.ignoreSitemap ? "skip" : "include",
deduplicateSimilarURLs: x.deduplicateSimilarURLs,
ignoreQueryParameters: x.ignoreQueryParameters,
regexOnFullURL: x.regexOnFullURL,
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 20aa1bbbd..c5e0f0356 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -356,78 +356,81 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
doc.metadata.url ?? doc.metadata.sourceURL ?? sc.originUrl!,
);
- const links = await crawler.filterLinks(
- await crawler.extractLinksFromHTML(
- rawHtml ?? "",
- doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
- ),
- Infinity,
- sc.crawlerOptions?.maxDepth ?? 10,
- );
- logger.debug("Discovered " + links.links.length + " links...", {
- linksLength: links.links.length,
- });
+ if (!sc.crawlerOptions?.sitemapOnly) {
+ const links = await crawler.filterLinks(
+ await crawler.extractLinksFromHTML(
+ rawHtml ?? "",
+ doc.metadata?.url ?? doc.metadata?.sourceURL ?? sc.originUrl!,
+ ),
+ Infinity,
+ sc.crawlerOptions?.maxDepth ?? 10,
+ );
+ logger.debug("Discovered " + links.links.length + " links...", {
+ linksLength: links.links.length,
+ });
- // Store robots blocked URLs in Redis set
- for (const [url, reason] of links.denialReasons) {
- if (reason === "URL blocked by robots.txt") {
- await recordRobotsBlocked(job.data.crawl_id, url);
+ // Store robots blocked URLs in Redis set
+ for (const [url, reason] of links.denialReasons) {
+ if (reason === "URL blocked by robots.txt") {
+ await recordRobotsBlocked(job.data.crawl_id, url);
+ }
}
- }
- for (const link of links.links) {
- if (await lockURL(job.data.crawl_id, sc, link)) {
- // This seems to work really welel
- const jobPriority = await getJobPriority({
- team_id: sc.team_id,
- basePriority: job.data.crawl_id ? 20 : 10,
- });
- const jobId = uuidv7();
-
- logger.debug(
- "Determined job priority " +
- jobPriority +
- " for URL " +
- JSON.stringify(link),
- { jobPriority, url: link },
- );
-
- await addScrapeJob(
- {
- url: link,
- mode: "single_urls",
+ for (const link of links.links) {
+ if (await lockURL(job.data.crawl_id, sc, link)) {
+ // This seems to work really welel
+ const jobPriority = await getJobPriority({
team_id: sc.team_id,
- scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
- internalOptions: sc.internalOptions,
- crawlerOptions: {
- ...sc.crawlerOptions,
- currentDiscoveryDepth:
- (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) + 1,
+ basePriority: job.data.crawl_id ? 20 : 10,
+ });
+ const jobId = uuidv7();
+
+ logger.debug(
+ "Determined job priority " +
+ jobPriority +
+ " for URL " +
+ JSON.stringify(link),
+ { jobPriority, url: link },
+ );
+
+ await addScrapeJob(
+ {
+ url: link,
+ mode: "single_urls",
+ team_id: sc.team_id,
+ scrapeOptions: scrapeOptions.parse(sc.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ crawlerOptions: {
+ ...sc.crawlerOptions,
+ currentDiscoveryDepth:
+ (job.data.crawlerOptions?.currentDiscoveryDepth ?? 0) +
+ 1,
+ },
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ requestId: job.data.requestId,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention: job.data.zeroDataRetention,
+ apiKeyId: job.data.apiKeyId,
},
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- requestId: job.data.requestId,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- apiKeyId: job.data.apiKeyId,
- },
- jobId,
- jobPriority,
- );
-
- await addCrawlJob(job.data.crawl_id, jobId, logger);
- logger.debug("Added job for URL " + JSON.stringify(link), {
- jobPriority,
- url: link,
- newJobId: jobId,
- });
- } else {
- // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
- // logger.debug("Could not lock URL " + JSON.stringify(link), {
- // url: link,
- // });
+ jobId,
+ jobPriority,
+ );
+
+ await addCrawlJob(job.data.crawl_id, jobId, logger);
+ logger.debug("Added job for URL " + JSON.stringify(link), {
+ jobPriority,
+ url: link,
+ newJobId: jobId,
+ });
+ } else {
+ // TODO: removed this, ok? too many 'not useful' logs (?) Mogery!
+ // logger.debug("Could not lock URL " + JSON.stringify(link), {
+ // url: link,
+ // });
+ }
}
}
@@ -719,7 +722,7 @@ async function kickoffGetIndexLinks(
crawler: WebCrawler,
url: string,
) {
- if (sc.crawlerOptions.ignoreSitemap) {
+ if (sc.crawlerOptions.ignoreSitemap || sc.crawlerOptions.sitemapOnly) {
return [];
}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 71e6d4e0a..02be36336 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -460,7 +460,7 @@ export interface CrawlOptions {
excludePaths?: string[] | null;
includePaths?: string[] | null;
maxDiscoveryDepth?: number | null;
- sitemap?: 'skip' | 'include';
+ sitemap?: 'skip' | 'include' | 'only';
ignoreQueryParameters?: boolean;
limit?: number | null;
crawlEntireDomain?: boolean;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 3ab4358b0..fcda4c526 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.13.1"
+__version__ = "4.13.2"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py
index 039870fb8..81ad03521 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_crawl_params.py
@@ -42,7 +42,7 @@ class TestCrawlParamsData:
assert data.include_paths is None
assert data.exclude_paths is None
assert data.max_discovery_depth is None
- assert data.ignore_sitemap is False
+ assert data.sitemap is None
assert data.limit is None
assert data.crawl_entire_domain is False
assert data.allow_external_links is False
@@ -67,4 +67,4 @@ class TestCrawlParamsData:
assert data.limit == 50
assert data.crawl_entire_domain is True
assert data.allow_external_links is False
- assert data.warning == "Test warning"
\ No newline at end of file
+ assert data.warning == "Test warning"
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 71e5b0bdd..c5df6d9d8 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -229,7 +229,8 @@ class FirecrawlClient:
exclude_paths: Optional[List[str]] = None,
include_paths: Optional[List[str]] = None,
max_discovery_depth: Optional[int] = None,
- ignore_sitemap: bool = False,
+ sitemap: Optional[Literal["only", "include", "skip"]] = None,
+ ignore_sitemap: Optional[bool] = None,
ignore_query_parameters: bool = False,
limit: Optional[int] = None,
crawl_entire_domain: bool = False,
@@ -254,7 +255,8 @@ class FirecrawlClient:
exclude_paths: Patterns of URLs to exclude
include_paths: Patterns of URLs to include
max_discovery_depth: Maximum depth for finding new URLs
- ignore_sitemap: Skip sitemap.xml processing
+ sitemap: Sitemap usage mode ("only" | "include" | "skip")
+ ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)
ignore_query_parameters: Ignore URL parameters
limit: Maximum pages to crawl
crawl_entire_domain: Follow parent directory links
@@ -277,25 +279,32 @@ class FirecrawlClient:
Exception: If the crawl fails to start or complete
TimeoutError: If timeout is reached
"""
- request = CrawlRequest(
- url=url,
- prompt=prompt,
- exclude_paths=exclude_paths,
- include_paths=include_paths,
- max_discovery_depth=max_discovery_depth,
- ignore_sitemap=ignore_sitemap,
- ignore_query_parameters=ignore_query_parameters,
- limit=limit,
- crawl_entire_domain=crawl_entire_domain,
- allow_external_links=allow_external_links,
- allow_subdomains=allow_subdomains,
- delay=delay,
- max_concurrency=max_concurrency,
- webhook=webhook,
- scrape_options=scrape_options,
- zero_data_retention=zero_data_retention,
- integration=integration,
- )
+ resolved_sitemap = sitemap
+ if resolved_sitemap is None and ignore_sitemap is not None:
+ resolved_sitemap = "skip" if ignore_sitemap else "include"
+
+ request_kwargs = {
+ "url": url,
+ "prompt": prompt,
+ "exclude_paths": exclude_paths,
+ "include_paths": include_paths,
+ "max_discovery_depth": max_discovery_depth,
+ "ignore_query_parameters": ignore_query_parameters,
+ "limit": limit,
+ "crawl_entire_domain": crawl_entire_domain,
+ "allow_external_links": allow_external_links,
+ "allow_subdomains": allow_subdomains,
+ "delay": delay,
+ "max_concurrency": max_concurrency,
+ "webhook": webhook,
+ "scrape_options": scrape_options,
+ "zero_data_retention": zero_data_retention,
+ "integration": integration,
+ }
+ if resolved_sitemap is not None:
+ request_kwargs["sitemap"] = resolved_sitemap
+
+ request = CrawlRequest(**request_kwargs)
return crawl_module.crawl(
self.http_client,
@@ -313,7 +322,8 @@ class FirecrawlClient:
exclude_paths: Optional[List[str]] = None,
include_paths: Optional[List[str]] = None,
max_discovery_depth: Optional[int] = None,
- ignore_sitemap: bool = False,
+ sitemap: Optional[Literal["only", "include", "skip"]] = None,
+ ignore_sitemap: Optional[bool] = None,
ignore_query_parameters: bool = False,
limit: Optional[int] = None,
crawl_entire_domain: bool = False,
@@ -335,7 +345,8 @@ class FirecrawlClient:
exclude_paths: Patterns of URLs to exclude
include_paths: Patterns of URLs to include
max_discovery_depth: Maximum depth for finding new URLs
- ignore_sitemap: Skip sitemap.xml processing
+ sitemap: Sitemap usage mode ("only" | "include" | "skip")
+ ignore_sitemap: Deprecated alias for sitemap ("skip" when true, "include" when false)
ignore_query_parameters: Ignore URL parameters
limit: Maximum pages to crawl
crawl_entire_domain: Follow parent directory links
@@ -354,25 +365,32 @@ class FirecrawlClient:
ValueError: If request is invalid
Exception: If the crawl operation fails to start
"""
- request = CrawlRequest(
- url=url,
- prompt=prompt,
- exclude_paths=exclude_paths,
- include_paths=include_paths,
- max_discovery_depth=max_discovery_depth,
- ignore_sitemap=ignore_sitemap,
- ignore_query_parameters=ignore_query_parameters,
- limit=limit,
- crawl_entire_domain=crawl_entire_domain,
- allow_external_links=allow_external_links,
- allow_subdomains=allow_subdomains,
- delay=delay,
- max_concurrency=max_concurrency,
- webhook=webhook,
- scrape_options=scrape_options,
- zero_data_retention=zero_data_retention,
- integration=integration,
- )
+ resolved_sitemap = sitemap
+ if resolved_sitemap is None and ignore_sitemap is not None:
+ resolved_sitemap = "skip" if ignore_sitemap else "include"
+
+ request_kwargs = {
+ "url": url,
+ "prompt": prompt,
+ "exclude_paths": exclude_paths,
+ "include_paths": include_paths,
+ "max_discovery_depth": max_discovery_depth,
+ "ignore_query_parameters": ignore_query_parameters,
+ "limit": limit,
+ "crawl_entire_domain": crawl_entire_domain,
+ "allow_external_links": allow_external_links,
+ "allow_subdomains": allow_subdomains,
+ "delay": delay,
+ "max_concurrency": max_concurrency,
+ "webhook": webhook,
+ "scrape_options": scrape_options,
+ "zero_data_retention": zero_data_retention,
+ "integration": integration,
+ }
+ if resolved_sitemap is not None:
+ request_kwargs["sitemap"] = resolved_sitemap
+
+ request = CrawlRequest(**request_kwargs)
return crawl_module.start_crawl(self.http_client, request)
@@ -977,4 +995,4 @@ class FirecrawlClient:
poll_interval=poll_interval,
timeout=wait_timeout,
)
-
\ No newline at end of file
+
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index ee20f34e3..c2493421f 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -81,6 +81,13 @@ class AsyncFirecrawlClient:
return await async_search.search(self.async_http_client, request)
async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
+ sitemap = kwargs.pop("sitemap", None)
+ ignore_sitemap = kwargs.pop("ignore_sitemap", None)
+ if sitemap is None and ignore_sitemap is not None:
+ sitemap = "skip" if ignore_sitemap else "include"
+ if sitemap is not None:
+ kwargs["sitemap"] = sitemap
+
request = CrawlRequest(url=url, **kwargs)
return await async_crawl.start_crawl(self.async_http_client, request)
@@ -414,4 +421,3 @@ class AsyncFirecrawlClient:
timeout: Optional[int] = None,
) -> AsyncWatcher:
return AsyncWatcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)
-
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
index 5d1d78830..c2a10d8d7 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/crawl.py
@@ -43,7 +43,7 @@ def _prepare_crawl_request(request: CrawlRequest) -> dict:
"include_paths": "includePaths",
"exclude_paths": "excludePaths",
"max_discovery_depth": "maxDiscoveryDepth",
- "ignore_sitemap": "ignoreSitemap",
+ "sitemap": "sitemap",
"ignore_query_parameters": "ignoreQueryParameters",
"crawl_entire_domain": "crawlEntireDomain",
"allow_external_links": "allowExternalLinks",
@@ -272,7 +272,7 @@ async def crawl_params_preview(client: AsyncHttpClient, request: CrawlParamsRequ
"includePaths": "include_paths",
"excludePaths": "exclude_paths",
"maxDiscoveryDepth": "max_discovery_depth",
- "ignoreSitemap": "ignore_sitemap",
+ "sitemap": "sitemap",
"ignoreQueryParameters": "ignore_query_parameters",
"crawlEntireDomain": "crawl_entire_domain",
"allowExternalLinks": "allow_external_links",
@@ -348,4 +348,3 @@ async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse:
"options": c.get("options"),
})
return ActiveCrawlsResponse(success=True, crawls=[ActiveCrawl(**nc) for nc in normalized])
-
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 673870939..297332a04 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -572,7 +572,7 @@ class CrawlRequest(BaseModel):
exclude_paths: Optional[List[str]] = None
include_paths: Optional[List[str]] = None
max_discovery_depth: Optional[int] = None
- sitemap: Literal["skip", "include"] = "include"
+ sitemap: Literal["skip", "include", "only"] = "include"
ignore_query_parameters: bool = False
limit: Optional[int] = None
crawl_entire_domain: bool = False
@@ -664,7 +664,7 @@ class CrawlParamsData(BaseModel):
include_paths: Optional[List[str]] = None
exclude_paths: Optional[List[str]] = None
max_discovery_depth: Optional[int] = None
- ignore_sitemap: bool = False
+ sitemap: Optional[Literal["skip", "include", "only"]] = None
ignore_query_parameters: bool = False
limit: Optional[int] = None
crawl_entire_domain: bool = False