Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(js-sdk,python-sdk): add threatProtection request option (#3966)

Gergő Móricz6d456aebb607
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index e19e01268..0442e591a 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.29.2",
+ "version": "4.30.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/threat-protection.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/threat-protection.unit.test.ts
new file mode 100644
index 000000000..7a238da09
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/threat-protection.unit.test.ts
@@ -0,0 +1,123 @@
+import { describe, test, expect, jest } from "@jest/globals";
+import { scrape } from "../../../v2/methods/scrape";
+import { startBatchScrape } from "../../../v2/methods/batch";
+import { startCrawl } from "../../../v2/methods/crawl";
+import { search } from "../../../v2/methods/search";
+import { map } from "../../../v2/methods/map";
+import { startExtract } from "../../../v2/methods/extract";
+import { startAgent } from "../../../v2/methods/agent";
+import type { ThreatProtectionOptions } from "../../../v2/types";
+
+const threatProtection: ThreatProtectionOptions = {
+ mode: "normal",
+ riskScoreThreshold: 80,
+ blacklist: ["*.blocked.example.com"],
+ whitelist: ["allowed.example.com"],
+ blockedTlds: ["zip"],
+ failurePolicy: "open",
+};
+
+function makeHttp(data: Record<string, unknown>) {
+ const post = jest.fn(async () => ({ status: 200, data }));
+ return {
+ post,
+ prepareHeaders: jest.fn(() => undefined),
+ } as any;
+}
+
+describe("v2 threatProtection request serialization", () => {
+ test("scrape sends threatProtection at top level", async () => {
+ const http = makeHttp({ success: true, data: {} });
+ await scrape(http, "https://example.com", { threatProtection });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/scrape",
+ expect.objectContaining({ url: "https://example.com", threatProtection }),
+ {},
+ );
+ });
+
+ test("batch scrape sends threatProtection at top level", async () => {
+ const http = makeHttp({ success: true, id: "job", url: "u" });
+ await startBatchScrape(http, ["https://example.com"], {
+ options: { threatProtection },
+ });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/batch/scrape",
+ expect.objectContaining({
+ urls: ["https://example.com"],
+ threatProtection,
+ }),
+ expect.anything(),
+ );
+ });
+
+ test("crawl sends threatProtection under scrapeOptions", async () => {
+ const http = makeHttp({ success: true, id: "job", url: "u" });
+ await startCrawl(http, {
+ url: "https://example.com",
+ scrapeOptions: { threatProtection },
+ });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/crawl",
+ expect.objectContaining({
+ scrapeOptions: expect.objectContaining({ threatProtection }),
+ }),
+ );
+ });
+
+ test("search sends threatProtection at top level and under scrapeOptions", async () => {
+ const http = makeHttp({ success: true, data: {} });
+ await search(http, {
+ query: "firecrawl",
+ threatProtection,
+ scrapeOptions: { threatProtection },
+ });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/search",
+ expect.objectContaining({
+ threatProtection,
+ scrapeOptions: expect.objectContaining({ threatProtection }),
+ }),
+ {},
+ );
+ });
+
+ test("map sends threatProtection at top level", async () => {
+ const http = makeHttp({ success: true, links: [] });
+ await map(http, "https://example.com", { threatProtection });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/map",
+ expect.objectContaining({ threatProtection }),
+ {},
+ );
+ });
+
+ test("extract sends threatProtection at top level", async () => {
+ const http = makeHttp({ success: true, id: "job" });
+ await startExtract(http, {
+ urls: ["https://example.com"],
+ prompt: "extract",
+ threatProtection,
+ });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/extract",
+ expect.objectContaining({ threatProtection }),
+ );
+ });
+
+ test("agent sends threatProtection at top level", async () => {
+ const http = makeHttp({ success: true, id: "job", status: "processing" });
+ await startAgent(http, { prompt: "find pricing", threatProtection });
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/agent",
+ expect.objectContaining({ threatProtection }),
+ );
+ });
+
+ test("threatProtection is omitted when not provided", async () => {
+ const http = makeHttp({ success: true, data: {} });
+ await scrape(http, "https://example.com", { onlyMainContent: true });
+ const body = http.post.mock.calls[0][1] as Record<string, unknown>;
+ expect(body).not.toHaveProperty("threatProtection");
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
index d789d66b3..08ae78563 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts
@@ -1,4 +1,4 @@
-import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";
+import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type ThreatProtectionOptions } from "../types";
import { HttpClient } from "../utils/httpClient";
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
@@ -14,6 +14,7 @@ function prepareAgentPayload(args: {
strictConstrainToURLs?: boolean;
model?: "spark-1-pro" | "spark-1-mini";
webhook?: string | AgentWebhookConfig;
+ threatProtection?: ThreatProtectionOptions;
}): Record<string, unknown> {
const body: Record<string, unknown> = {};
if (args.urls) body.urls = args.urls;
@@ -27,6 +28,8 @@ function prepareAgentPayload(args: {
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
if (args.model !== null && args.model !== undefined) body.model = args.model;
if (args.webhook != null) body.webhook = args.webhook;
+ if (args.threatProtection != null)
+ body.threatProtection = args.threatProtection;
return body;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
index 84e59218f..84a11e610 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/extract.ts
@@ -1,4 +1,4 @@
-import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type WebhookConfig } from "../types";
+import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type ThreatProtectionOptions, type WebhookConfig } from "../types";
import { HttpClient } from "../utils/httpClient";
import { ensureValidScrapeOptions } from "../utils/validation";
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
@@ -19,6 +19,7 @@ function prepareExtractPayload(args: {
origin?: string;
agent?: AgentOptions;
webhook?: string | WebhookConfig | null;
+ threatProtection?: ThreatProtectionOptions;
}): Record<string, unknown> {
const body: Record<string, unknown> = {};
if (args.urls) body.urls = args.urls;
@@ -35,6 +36,8 @@ function prepareExtractPayload(args: {
if (args.origin) body.origin = args.origin;
if (args.agent) body.agent = args.agent;
if (args.webhook != null) body.webhook = args.webhook;
+ if (args.threatProtection != null)
+ body.threatProtection = args.threatProtection;
if (args.scrapeOptions) {
ensureValidScrapeOptions(args.scrapeOptions);
body.scrapeOptions = args.scrapeOptions;
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/map.ts b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
index 5a887ca82..eab0b6d06 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/map.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/map.ts
@@ -24,6 +24,8 @@ function prepareMapPayload(
payload.integration = options.integration.trim();
if (options.origin) payload.origin = options.origin;
if (options.location != null) payload.location = options.location;
+ if (options.threatProtection != null)
+ payload.threatProtection = options.threatProtection;
}
return payload;
}
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 975227ee0..d1610959d 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -41,6 +41,8 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
payload.integration = req.integration.trim();
if (req.origin) payload.origin = req.origin;
if (req.enterprise) payload.enterprise = req.enterprise;
+ if (req.threatProtection != null)
+ payload.threatProtection = req.threatProtection;
if (req.scrapeOptions) {
ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
payload.scrapeOptions = req.scrapeOptions;
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 117b8ad32..c75b174c9 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -208,6 +208,7 @@ export interface ScrapeOptions {
storeInCache?: boolean;
lockdown?: boolean;
redactPII?: boolean | RedactPIIOptions;
+ threatProtection?: ThreatProtectionOptions;
profile?: {
name: string;
saveChanges?: boolean;
@@ -241,6 +242,27 @@ export interface RedactPIIOptions {
replaceStyle?: "tag" | "mask" | "remove";
}
+/**
+ * Enterprise: per-request field-level override of your team's threat
+ * protection policy. Requires threat protection to be enabled for your team
+ * and request overrides to be allowed in the team configuration. Only the
+ * fields you provide replace the team policy's values.
+ */
+export interface ThreatProtectionOptions {
+ /** "off" disables scanning for this request; "normal" applies the policy. */
+ mode?: "off" | "normal";
+ /** Block verdicts at or above this risk score (integer 0-100). */
+ riskScoreThreshold?: number;
+ /** Exact domains or globs like "*.example.com" to always block (max 1000). */
+ blacklist?: string[];
+ /** Exact domains or globs to always allow; wins over everything (max 1000). */
+ whitelist?: string[];
+ /** Lowercase TLDs without the leading dot, e.g. "zip" (max 1000). */
+ blockedTlds?: string[];
+ /** Behavior when scanning is unavailable: "closed" blocks, "open" allows. */
+ failurePolicy?: "open" | "closed";
+}
+
export type ParseFileData =
| Blob
| File
@@ -267,6 +289,7 @@ export type ParseOptions = Omit<
| "storeInCache"
| "lockdown"
| "proxy"
+ | "threatProtection"
> & {
formats?: ParseFormatOption[];
proxy?: "basic" | "auto";
@@ -681,6 +704,7 @@ export interface SearchRequest {
* your team.
*/
enterprise?: Array<"default" | "anon" | "zdr">;
+ threatProtection?: ThreatProtectionOptions;
integration?: string;
origin?: string;
}
@@ -769,6 +793,7 @@ export interface MapOptions {
integration?: string;
origin?: string;
location?: LocationConfig;
+ threatProtection?: ThreatProtectionOptions;
}
export type FeedbackRating = "good" | "partial" | "bad";
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 2eb9e5f0b..64a6d45f8 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.31.0"
+__version__ = "4.32.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_threat_protection_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_threat_protection_request_preparation.py
new file mode 100644
index 000000000..1e59f78dc
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_threat_protection_request_preparation.py
@@ -0,0 +1,87 @@
+from firecrawl.v2.types import (
+ CrawlRequest,
+ MapOptions,
+ ScrapeOptions,
+ SearchRequest,
+ ThreatProtectionOptions,
+)
+from firecrawl.v2.methods.aio.agent import _prepare_agent_request
+from firecrawl.v2.methods.aio.batch import _prepare as _prepare_batch_request
+from firecrawl.v2.methods.aio.crawl import _prepare_crawl_request
+from firecrawl.v2.methods.aio.extract import _prepare_extract_request
+from firecrawl.v2.methods.aio.map import _prepare_map_request
+from firecrawl.v2.methods.aio.search import _prepare_search_request
+
+
+EXPECTED_WIRE_PAYLOAD = {
+ "mode": "normal",
+ "riskScoreThreshold": 80,
+ "blacklist": ["*.blocked.example.com"],
+ "whitelist": ["allowed.example.com"],
+ "blockedTlds": ["zip"],
+ "failurePolicy": "open",
+}
+
+
+def _threat_protection() -> ThreatProtectionOptions:
+ return ThreatProtectionOptions(
+ mode="normal",
+ risk_score_threshold=80,
+ blacklist=["*.blocked.example.com"],
+ whitelist=["allowed.example.com"],
+ blocked_tlds=["zip"],
+ failure_policy="open",
+ )
+
+
+class TestAioThreatProtectionRequestPreparation:
+ """The aio request preparers must serialize threat_protection too."""
+
+ def test_batch_scrape_top_level(self):
+ data = _prepare_batch_request(
+ ["https://example.com"],
+ options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_crawl_scrape_options(self):
+ request = CrawlRequest(
+ url="https://example.com",
+ scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ data = _prepare_crawl_request(request)
+ assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+ def test_search_top_level_and_scrape_options(self):
+ request = SearchRequest(
+ query="firecrawl",
+ threat_protection=_threat_protection(),
+ scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ data = _prepare_search_request(request)
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_map_top_level(self):
+ options = MapOptions(threat_protection=_threat_protection())
+ data = _prepare_map_request("https://example.com", options)
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_extract_top_level(self):
+ data = _prepare_extract_request(
+ ["https://example.com"],
+ prompt="extract",
+ threat_protection=_threat_protection(),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+ def test_agent_top_level(self):
+ data = _prepare_agent_request(
+ None,
+ prompt="find pricing",
+ threat_protection=_threat_protection(),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_threat_protection_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_threat_protection_request_preparation.py
new file mode 100644
index 000000000..7b4dce175
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_threat_protection_request_preparation.py
@@ -0,0 +1,118 @@
+from firecrawl.v2.types import (
+ CrawlRequest,
+ MapOptions,
+ ScrapeOptions,
+ SearchRequest,
+ ThreatProtectionOptions,
+)
+from firecrawl.v2.methods.agent import _prepare_agent_request
+from firecrawl.v2.methods.batch import prepare_batch_scrape_request
+from firecrawl.v2.methods.crawl import _prepare_crawl_request
+from firecrawl.v2.methods.extract import _prepare_extract_request
+from firecrawl.v2.methods.map import _prepare_map_request
+from firecrawl.v2.methods.scrape import _prepare_scrape_request
+from firecrawl.v2.methods.search import _prepare_search_request
+
+
+EXPECTED_WIRE_PAYLOAD = {
+ "mode": "normal",
+ "riskScoreThreshold": 80,
+ "blacklist": ["*.blocked.example.com"],
+ "whitelist": ["allowed.example.com"],
+ "blockedTlds": ["zip"],
+ "failurePolicy": "open",
+}
+
+
+def _threat_protection() -> ThreatProtectionOptions:
+ return ThreatProtectionOptions(
+ mode="normal",
+ risk_score_threshold=80,
+ blacklist=["*.blocked.example.com"],
+ whitelist=["allowed.example.com"],
+ blocked_tlds=["zip"],
+ failure_policy="open",
+ )
+
+
+class TestThreatProtectionRequestPreparation:
+ """threat_protection must reach the wire body camelCased everywhere."""
+
+ def test_construction_by_alias(self):
+ options = ThreatProtectionOptions(
+ riskScoreThreshold=42,
+ blockedTlds=["zip"],
+ failurePolicy="closed",
+ )
+ assert options.risk_score_threshold == 42
+ assert options.blocked_tlds == ["zip"]
+ assert options.failure_policy == "closed"
+
+ def test_scrape_top_level(self):
+ options = ScrapeOptions(threat_protection=_threat_protection())
+ data = _prepare_scrape_request("https://example.com", options)
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_scrape_omitted_when_unset(self):
+ data = _prepare_scrape_request(
+ "https://example.com", ScrapeOptions(only_main_content=True)
+ )
+ assert "threatProtection" not in data
+ assert "threat_protection" not in data
+
+ def test_batch_scrape_top_level(self):
+ data = prepare_batch_scrape_request(
+ ["https://example.com"],
+ options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_crawl_scrape_options(self):
+ request = CrawlRequest(
+ url="https://example.com",
+ scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ data = _prepare_crawl_request(request)
+ assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+ def test_search_top_level_and_scrape_options(self):
+ request = SearchRequest(
+ query="firecrawl",
+ threat_protection=_threat_protection(),
+ scrape_options=ScrapeOptions(threat_protection=_threat_protection()),
+ )
+ data = _prepare_search_request(request)
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert data["scrapeOptions"]["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_map_top_level(self):
+ options = MapOptions(threat_protection=_threat_protection())
+ data = _prepare_map_request("https://example.com", options)
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+ assert "threat_protection" not in data
+
+ def test_extract_top_level(self):
+ data = _prepare_extract_request(
+ ["https://example.com"],
+ prompt="extract",
+ threat_protection=_threat_protection(),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+ def test_agent_top_level(self):
+ data = _prepare_agent_request(
+ None,
+ prompt="find pricing",
+ threat_protection=_threat_protection(),
+ )
+ assert data["threatProtection"] == EXPECTED_WIRE_PAYLOAD
+
+ def test_partial_override_only_sends_provided_fields(self):
+ options = ScrapeOptions(
+ threat_protection=ThreatProtectionOptions(mode="off")
+ )
+ data = _prepare_scrape_request("https://example.com", options)
+ assert data["threatProtection"] == {"mode": "off"}
diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.py
index 4bb832409..79dc613fe 100644
--- a/apps/python-sdk/firecrawl/types.py
+++ b/apps/python-sdk/firecrawl/types.py
@@ -19,6 +19,7 @@ from .v2.types import (
ScrapeRequest,
ScrapeData,
ScrapeResponse,
+ ThreatProtectionOptions,
# Crawl types
CrawlRequest,
@@ -102,6 +103,7 @@ __all__ = [
'ScrapeRequest',
'ScrapeData',
'ScrapeResponse',
+ 'ThreatProtectionOptions',
# Crawl types
'CrawlRequest',
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 08222a13d..b04810290 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -42,6 +42,7 @@ from .types import (
Location,
PaginationConfig,
AgentOptions,
+ ThreatProtectionOptions,
Monitor,
MonitorCheck,
MonitorCheckDetail,
@@ -76,7 +77,8 @@ _SCRAPE_OPTION_KEYS = frozenset({
"only_main_content", "timeout", "wait_for", "mobile",
"parsers", "actions", "location", "skip_tls_verification",
"remove_base64_images", "fast_mode", "use_mock", "block_ads",
- "proxy", "max_age", "store_in_cache", "lockdown", "profile",
+ "proxy", "max_age", "store_in_cache", "lockdown", "threat_protection",
+ "profile",
})
@@ -157,6 +159,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
integration: Optional[str] = None,
) -> Document:
@@ -184,6 +187,7 @@ class FirecrawlClient:
max_age: Maximum age of the cache
store_in_cache: Whether to store the result in the cache
lockdown: Serve only previously cached results; never make outbound requests. Returns 404 SCRAPE_LOCKDOWN_CACHE_MISS on cache miss.
+ threat_protection: Enterprise per-request override of the team's threat protection policy
profile: Browser profile for persistent state (e.g. {"name": "my-profile", "saveChanges": True})
Returns:
Document
@@ -210,10 +214,11 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ threat_protection=threat_protection,
profile=profile,
integration=integration,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, profile, integration]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, integration]) else None
return scrape_module.scrape(self.http_client, url, options)
def search_papers(self, query: str, **kwargs):
@@ -393,6 +398,7 @@ class FirecrawlClient:
scrape_options: Optional[ScrapeOptions] = None,
integration: Optional[str] = None,
enterprise: Optional[List[str]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> SearchData:
"""
Search for documents.
@@ -407,6 +413,8 @@ class FirecrawlClient:
enterprise: Enterprise search options. Use ["zdr"] for end-to-end
Zero Data Retention or ["anon"] for anonymized search. Must be
enabled for your team.
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
SearchData containing the search results
@@ -425,6 +433,7 @@ class FirecrawlClient:
scrape_options=scrape_options,
integration=integration,
enterprise=enterprise,
+ threat_protection=threat_protection,
)
return search_module.search(self.http_client, request)
@@ -468,6 +477,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
@@ -517,6 +527,7 @@ class FirecrawlClient:
max_age: Cache max age (convenience kwarg)
store_in_cache: Cache results (convenience kwarg)
lockdown: Serve only cached results (convenience kwarg)
+ threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
@@ -542,7 +553,7 @@ class FirecrawlClient:
remove_base64_images=remove_base64_images, fast_mode=fast_mode,
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
- profile=profile,
+ threat_protection=threat_protection, profile=profile,
).items() if v is not None}
if scrape_kwargs:
scrape_options = ScrapeOptions(**scrape_kwargs)
@@ -625,6 +636,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
profile: Optional[Dict[str, Any]] = None,
regex_on_full_url: bool = False,
deduplicate_similar_urls: bool = True,
@@ -672,6 +684,7 @@ class FirecrawlClient:
max_age: Cache max age (convenience kwarg)
store_in_cache: Cache results (convenience kwarg)
lockdown: Serve only cached results (convenience kwarg)
+ threat_protection: Enterprise threat protection override (convenience kwarg)
profile: Browser profile (convenience kwarg)
regex_on_full_url: Apply includePaths/excludePaths regex to the full URL (including query parameters) instead of just the pathname
deduplicate_similar_urls: Whether to deduplicate similar URLs during crawl (default: True)
@@ -694,7 +707,7 @@ class FirecrawlClient:
remove_base64_images=remove_base64_images, fast_mode=fast_mode,
use_mock=use_mock, block_ads=block_ads, proxy=proxy,
max_age=max_age, store_in_cache=store_in_cache, lockdown=lockdown,
- profile=profile,
+ threat_protection=threat_protection, profile=profile,
).items() if v is not None}
if scrape_kwargs:
scrape_options = ScrapeOptions(**scrape_kwargs)
@@ -825,6 +838,7 @@ class FirecrawlClient:
timeout: Optional[int] = None,
integration: Optional[str] = None,
location: Optional[Location] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> MapData:
"""Map a URL and return discovered links.
@@ -836,6 +850,8 @@ class FirecrawlClient:
limit: Maximum number of links to return
sitemap: Sitemap usage mode ("only" | "include" | "skip")
timeout: Request timeout in milliseconds
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
MapData containing the discovered links
@@ -848,8 +864,9 @@ class FirecrawlClient:
sitemap=sitemap if sitemap is not None else "include",
timeout=timeout,
integration=integration,
- location=location
- ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location]) else None
+ location=location,
+ threat_protection=threat_protection
+ ) if any(v is not None for v in [search, include_subdomains, ignore_query_parameters, limit, sitemap, timeout, integration, location, threat_protection]) else None
return map_module.map(self.http_client, url, options)
@@ -1006,6 +1023,7 @@ class FirecrawlClient:
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
agent: Optional[AgentOptions] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Start an extract job (non-blocking).
@@ -1026,6 +1044,8 @@ class FirecrawlClient:
ignore_invalid_urls: Skip invalid URLs instead of failing
integration: Integration tag/name
agent: Agent configuration
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
Response payload with job id/status (poll with get_extract_status)
"""
@@ -1042,6 +1062,7 @@ class FirecrawlClient:
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
agent=agent,
+ threat_protection=threat_protection,
)
def extract(
@@ -1060,6 +1081,7 @@ class FirecrawlClient:
timeout: Optional[int] = None,
integration: Optional[str] = None,
agent: Optional[AgentOptions] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Extract structured data and wait until completion.
@@ -1082,6 +1104,8 @@ class FirecrawlClient:
timeout: Maximum seconds to wait (None for no timeout)
integration: Integration tag/name
agent: Agent configuration
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
Final extract response when completed
"""
@@ -1100,6 +1124,7 @@ class FirecrawlClient:
timeout=timeout,
integration=integration,
agent=agent,
+ threat_protection=threat_protection,
)
def start_batch_scrape(
@@ -1126,6 +1151,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -1158,6 +1184,7 @@ class FirecrawlClient:
max_age: Cache max age
store_in_cache: Whether to store results in cache
lockdown: Serve only previously cached results; never make outbound requests.
+ threat_protection: Enterprise per-request override of the team's threat protection policy
webhook: Webhook configuration
append_to_id: Append to an existing batch job
ignore_invalid_urls: Skip invalid URLs without failing
@@ -1191,8 +1218,9 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ threat_protection=threat_protection,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None
return batch_module.start_batch_scrape(
self.http_client,
@@ -1297,6 +1325,7 @@ class FirecrawlClient:
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Start an agent job (non-blocking).
@@ -1308,6 +1337,8 @@ class FirecrawlClient:
max_credits: Maximum credits to use (optional)
model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")
webhook: Webhook URL or configuration for notifications
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
Response payload with job id/status (poll with get_agent_status)
"""
@@ -1321,6 +1352,7 @@ class FirecrawlClient:
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
def agent(
@@ -1336,6 +1368,7 @@ class FirecrawlClient:
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Run an agent and wait until completion.
@@ -1349,6 +1382,8 @@ class FirecrawlClient:
max_credits: Maximum credits to use (optional)
model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")
webhook: Webhook URL or configuration for notifications
+ threat_protection: Enterprise per-request override of the team's
+ threat protection policy
Returns:
Final agent response when completed
"""
@@ -1364,6 +1399,7 @@ class FirecrawlClient:
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
def get_agent_status(self, job_id: str):
@@ -1542,6 +1578,7 @@ class FirecrawlClient:
max_age: Optional[int] = None,
store_in_cache: Optional[bool] = None,
lockdown: Optional[bool] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
webhook: Optional[Union[str, WebhookConfig]] = None,
append_to_id: Optional[str] = None,
ignore_invalid_urls: Optional[bool] = None,
@@ -1577,8 +1614,9 @@ class FirecrawlClient:
max_age=max_age,
store_in_cache=store_in_cache,
lockdown=lockdown,
+ threat_protection=threat_protection,
).items() if v is not None}
- ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown]) else None
+ ) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection]) else None
return batch_module.batch_scrape(
self.http_client,
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 20e8184be..00fa01757 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -37,6 +37,7 @@ from .types import (
PDFAction,
Location,
PaginationConfig,
+ ThreatProtectionOptions,
Monitor,
MonitorCheck,
MonitorCheckDetail,
@@ -397,6 +398,7 @@ class AsyncFirecrawlClient:
sitemap: Optional[Literal["only", "include", "skip"]] = None,
timeout: Optional[int] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> MapData:
options = MapOptions(
search=search,
@@ -405,7 +407,8 @@ class AsyncFirecrawlClient:
sitemap=sitemap if sitemap is not None else "include",
timeout=timeout,
integration=integration,
- ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout]) else None
+ threat_protection=threat_protection,
+ ) if any(v is not None for v in [search, include_subdomains, limit, sitemap, integration, timeout, threat_protection]) else None
return await async_map.map(self.async_http_client, url, options)
async def create_monitor(
@@ -589,6 +592,7 @@ class AsyncFirecrawlClient:
poll_interval: int = 2,
timeout: Optional[int] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Extract structured data and wait until completion (async).
@@ -611,6 +615,7 @@ class AsyncFirecrawlClient:
poll_interval=poll_interval,
timeout=timeout,
integration=integration,
+ threat_protection=threat_protection,
)
async def get_extract_status(self, job_id: str):
@@ -636,6 +641,7 @@ class AsyncFirecrawlClient:
scrape_options: Optional['ScrapeOptions'] = None,
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
"""Start an extract job (non-blocking, async).
@@ -656,6 +662,7 @@ class AsyncFirecrawlClient:
scrape_options=scrape_options,
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
+ threat_protection=threat_protection,
)
# Agent
@@ -672,6 +679,7 @@ class AsyncFirecrawlClient:
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
return await async_agent.agent(
self.async_http_client,
@@ -685,6 +693,7 @@ class AsyncFirecrawlClient:
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
async def get_agent_status(self, job_id: str):
@@ -701,6 +710,7 @@ class AsyncFirecrawlClient:
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
):
return await async_agent.start_agent(
self.async_http_client,
@@ -712,6 +722,7 @@ class AsyncFirecrawlClient:
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
async def cancel_agent(self, job_id: str) -> bool:
diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.py
index 00169c4e9..e6bc6c412 100644
--- a/apps/python-sdk/firecrawl/v2/methods/agent.py
+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py
@@ -1,7 +1,7 @@
from typing import Any, Dict, List, Literal, Optional, Union
import time
-from ..types import AgentResponse, AgentWebhookConfig
+from ..types import AgentResponse, AgentWebhookConfig, ThreatProtectionOptions
from ..utils.http_client import HttpClient
from ..utils.error_handler import handle_response_error
from ..utils.validation import _normalize_schema
@@ -17,6 +17,7 @@ def _prepare_agent_request(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -44,6 +45,10 @@ def _prepare_agent_request(
body["webhook"] = webhook
else:
body["webhook"] = webhook.model_dump(exclude_none=True)
+ if threat_protection is not None:
+ body["threatProtection"] = threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
return body
@@ -67,6 +72,7 @@ def start_agent(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> AgentResponse:
body = _prepare_agent_request(
urls,
@@ -77,6 +83,7 @@ def start_agent(
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
resp = client.post("/v2/agent", body)
if not resp.ok:
@@ -123,6 +130,7 @@ def agent(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> AgentResponse:
started = start_agent(
client,
@@ -134,6 +142,7 @@ def agent(
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
job_id = getattr(started, "id", None)
if not job_id:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
index 6190f1f18..601399871 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py
@@ -1,7 +1,8 @@
from typing import Any, Dict, List, Literal, Optional, Union
import asyncio
-from ...types import AgentResponse, AgentWebhookConfig
+from ...types import AgentResponse, AgentWebhookConfig, ThreatProtectionOptions
+from ...utils.error_handler import handle_response_error
from ...utils.http_client_async import AsyncHttpClient
from ...utils.validation import _normalize_schema
@@ -16,6 +17,7 @@ def _prepare_agent_request(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -43,6 +45,10 @@ def _prepare_agent_request(
body["webhook"] = webhook
else:
body["webhook"] = webhook.model_dump(exclude_none=True)
+ if threat_protection is not None:
+ body["threatProtection"] = threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
return body
@@ -66,6 +72,7 @@ async def start_agent(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> AgentResponse:
body = _prepare_agent_request(
urls,
@@ -76,14 +83,19 @@ async def start_agent(
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
resp = await client.post("/v2/agent", body)
+ if not resp.ok:
+ handle_response_error(resp, "agent")
payload = _normalize_agent_response_payload(resp.json())
return AgentResponse(**payload)
async def get_agent_status(client: AsyncHttpClient, job_id: str) -> AgentResponse:
resp = await client.get(f"/v2/agent/{job_id}")
+ if not resp.ok:
+ handle_response_error(resp, "agent-status")
payload = _normalize_agent_response_payload(resp.json())
return AgentResponse(**payload)
@@ -118,6 +130,7 @@ async def agent(
strict_constrain_to_urls: Optional[bool] = None,
model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,
webhook: Optional[Union[str, AgentWebhookConfig]] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> AgentResponse:
started = await start_agent(
client,
@@ -129,6 +142,7 @@ async def agent(
strict_constrain_to_urls=strict_constrain_to_urls,
model=model,
webhook=webhook,
+ threat_protection=threat_protection,
)
job_id = getattr(started, "id", None)
if not job_id:
@@ -151,4 +165,6 @@ async def cancel_agent(client: AsyncHttpClient, job_id: str) -> bool:
Exception: If the cancellation fails
"""
resp = await client.delete(f"/v2/agent/{job_id}")
+ if not resp.ok:
+ handle_response_error(resp, "cancel agent")
return resp.json().get("success", False)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/extract.py b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
index f9dfda2c2..6802fe4f5 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/extract.py
@@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional
import asyncio
import warnings
-from ...types import ExtractResponse, ScrapeOptions
+from ...types import ExtractResponse, ScrapeOptions, ThreatProtectionOptions
from ...utils.http_client_async import AsyncHttpClient
from ...utils.validation import prepare_scrape_options
@@ -25,6 +25,7 @@ def _prepare_extract_request(
scrape_options: Optional[ScrapeOptions] = None,
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -49,6 +50,10 @@ def _prepare_extract_request(
body["scrapeOptions"] = prepared
if integration is not None and str(integration).strip():
body["integration"] = str(integration).strip()
+ if threat_protection is not None:
+ body["threatProtection"] = threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
return body
@@ -65,6 +70,7 @@ async def start_extract(
scrape_options: Optional[ScrapeOptions] = None,
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> ExtractResponse:
"""Start an extract job (non-blocking, async).
@@ -85,6 +91,7 @@ async def start_extract(
scrape_options=scrape_options,
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
+ threat_protection=threat_protection,
)
resp = await client.post("/v2/extract", body)
return ExtractResponse(**resp.json())
@@ -135,6 +142,7 @@ async def extract(
poll_interval: int = 2,
timeout: Optional[int] = None,
integration: Optional[str] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> ExtractResponse:
"""Extract structured data and wait until completion (async).
@@ -156,6 +164,7 @@ async def extract(
scrape_options=scrape_options,
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
+ threat_protection=threat_protection,
)
job_id = getattr(started, "id", None)
if not job_id:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/map.py b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
index 48f994dbb..883b01cb5 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/map.py
@@ -28,6 +28,10 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["integration"] = options.integration.strip()
if options.location is not None:
data["location"] = options.location.model_dump(exclude_none=True)
+ if options.threat_protection is not None:
+ data["threatProtection"] = options.threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
payload.update(data)
return payload
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
index 54c5e0d39..02f5e0bb0 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
@@ -166,6 +166,12 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
data["excludeDomains"] = validated_request.exclude_domains
data.pop("exclude_domains", None)
+ if validated_request.threat_protection is not None:
+ data["threatProtection"] = validated_request.threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
+ data.pop("threat_protection", None)
+
if validated_request.scrape_options is not None:
scrape_data = prepare_scrape_options(validated_request.scrape_options)
if scrape_data:
diff --git a/apps/python-sdk/firecrawl/v2/methods/extract.py b/apps/python-sdk/firecrawl/v2/methods/extract.py
index 73a7ce5c6..e6c0bd4e0 100644
--- a/apps/python-sdk/firecrawl/v2/methods/extract.py
+++ b/apps/python-sdk/firecrawl/v2/methods/extract.py
@@ -2,7 +2,7 @@ from typing import Any, Dict, List, Optional
import time
import warnings
-from ..types import ExtractResponse, ScrapeOptions
+from ..types import ExtractResponse, ScrapeOptions, ThreatProtectionOptions
from ..types import AgentOptions
from ..utils.http_client import HttpClient
from ..utils.validation import prepare_scrape_options
@@ -28,6 +28,7 @@ def _prepare_extract_request(
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
agent: Optional[AgentOptions] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> Dict[str, Any]:
body: Dict[str, Any] = {}
if urls is not None:
@@ -57,6 +58,10 @@ def _prepare_extract_request(
body["agent"] = agent.model_dump(exclude_none=True) # type: ignore[attr-defined]
except AttributeError:
body["agent"] = agent # fallback
+ if threat_protection is not None:
+ body["threatProtection"] = threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
return body
@@ -85,6 +90,7 @@ def start_extract(
ignore_invalid_urls: Optional[bool] = None,
integration: Optional[str] = None,
agent: Optional[AgentOptions] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> ExtractResponse:
"""Start an extract job (non-blocking).
@@ -106,6 +112,7 @@ def start_extract(
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
agent=agent,
+ threat_protection=threat_protection,
)
resp = client.post("/v2/extract", body)
if not resp.ok:
@@ -163,6 +170,7 @@ def extract(
timeout: Optional[int] = None,
integration: Optional[str] = None,
agent: Optional[AgentOptions] = None,
+ threat_protection: Optional[ThreatProtectionOptions] = None,
) -> ExtractResponse:
"""Extract structured data and wait until completion.
@@ -185,6 +193,7 @@ def extract(
ignore_invalid_urls=ignore_invalid_urls,
integration=integration,
agent=agent,
+ threat_protection=threat_protection,
)
job_id = getattr(started, "id", None)
if not job_id:
diff --git a/apps/python-sdk/firecrawl/v2/methods/map.py b/apps/python-sdk/firecrawl/v2/methods/map.py
index 686558a25..5181931cf 100644
--- a/apps/python-sdk/firecrawl/v2/methods/map.py
+++ b/apps/python-sdk/firecrawl/v2/methods/map.py
@@ -35,6 +35,10 @@ def _prepare_map_request(url: str, options: Optional[MapOptions] = None) -> Dict
data["integration"] = options.integration.strip()
if options.location is not None:
data["location"] = options.location.model_dump(exclude_none=True)
+ if options.threat_protection is not None:
+ data["threatProtection"] = options.threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
payload.update(data)
return payload
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index 118496f09..7e6e30b0a 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -201,6 +201,13 @@ def _prepare_search_request(request: SearchRequest) -> Dict[str, Any]:
data["excludeDomains"] = validated_request.exclude_domains
data.pop("exclude_domains", None)
+ # threat_protection → threatProtection
+ if validated_request.threat_protection is not None:
+ data["threatProtection"] = validated_request.threat_protection.model_dump(
+ by_alias=True, exclude_none=True
+ )
+ data.pop("threat_protection", None)
+
# scrape_options → scrapeOptions
if validated_request.scrape_options is not None:
scrape_data = prepare_scrape_options(validated_request.scrape_options)
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 8d766d4a0..7b3e043af 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -749,6 +749,35 @@ class RedactPIIOptions(BaseModel):
model_config = {"populate_by_name": True}
+class ThreatProtectionOptions(BaseModel):
+ """Enterprise: per-request field-level override of your team's threat
+ protection policy.
+
+ Requires threat protection to be enabled for your team and request
+ overrides to be allowed in the team configuration. Only the fields you
+ explicitly provide replace the team policy's values.
+ """
+
+ # "off" disables scanning for this request; "normal" applies the policy.
+ mode: Optional[Literal["off", "normal"]] = None
+ # Block verdicts at or above this risk score (integer 0-100).
+ risk_score_threshold: Optional[int] = Field(
+ default=None, alias="riskScoreThreshold"
+ )
+ # Exact domains or globs like "*.example.com" to always block (max 1000).
+ blacklist: Optional[List[str]] = None
+ # Exact domains or globs to always allow; wins over everything (max 1000).
+ whitelist: Optional[List[str]] = None
+ # Lowercase TLDs without the leading dot, e.g. "zip" (max 1000).
+ blocked_tlds: Optional[List[str]] = Field(default=None, alias="blockedTlds")
+ # Behavior when scanning is unavailable: "closed" blocks, "open" allows.
+ failure_policy: Optional[Literal["open", "closed"]] = Field(
+ default=None, alias="failurePolicy"
+ )
+
+ model_config = {"populate_by_name": True}
+
+
class ScrapeOptions(BaseModel):
"""Options for scraping operations."""
@@ -790,6 +819,9 @@ class ScrapeOptions(BaseModel):
redact_pii: Optional[Union[bool, RedactPIIOptions]] = Field(
default=None, alias="redactPII"
)
+ threat_protection: Optional[ThreatProtectionOptions] = Field(
+ default=None, alias="threatProtection"
+ )
profile: Optional[Dict[str, Any]] = None
integration: Optional[str] = None
@@ -1021,6 +1053,7 @@ class MapOptions(BaseModel):
timeout: Optional[int] = None
integration: Optional[str] = None
location: Optional["Location"] = None
+ threat_protection: Optional[ThreatProtectionOptions] = None
class MapRequest(BaseModel):
@@ -1573,6 +1606,7 @@ class SearchRequest(BaseModel):
# Enterprise search options. Use ["zdr"] for end-to-end Zero Data
# Retention or ["anon"] for anonymized search. Must be enabled for your team.
enterprise: Optional[List[str]] = None
+ threat_protection: Optional[ThreatProtectionOptions] = None
integration: Optional[str] = None
@field_validator("sources")
diff --git a/apps/python-sdk/firecrawl/v2/utils/validation.py b/apps/python-sdk/firecrawl/v2/utils/validation.py
index e5e8e9c56..d4d17ae04 100644
--- a/apps/python-sdk/firecrawl/v2/utils/validation.py
+++ b/apps/python-sdk/firecrawl/v2/utils/validation.py
@@ -567,6 +567,7 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
"store_in_cache": "storeInCache",
"max_age": "maxAge",
"redact_pii": "redactPII",
+ "threat_protection": "threatProtection",
}
# Apply field mappings
@@ -581,6 +582,18 @@ def prepare_scrape_options(options: Optional[ScrapeOptions]) -> Optional[Dict[st
scrape_data["redactPII"]["replaceStyle"] = scrape_data["redactPII"].pop(
"replace_style"
)
+
+ # threatProtection is a nested object whose inner fields also need
+ # camel-casing.
+ if isinstance(scrape_data.get("threatProtection"), dict):
+ threat_data = scrape_data["threatProtection"]
+ for snake_case, camel_case in (
+ ("risk_score_threshold", "riskScoreThreshold"),
+ ("blocked_tlds", "blockedTlds"),
+ ("failure_policy", "failurePolicy"),
+ ):
+ if snake_case in threat_data:
+ threat_data[camel_case] = threat_data.pop(snake_case)
# Handle special cases
for key, value in options_data.items():
diff --git a/apps/python-sdk/tests/test_crawl_direct_kwargs.py b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
index 726dfaf52..087a7f195 100644
--- a/apps/python-sdk/tests/test_crawl_direct_kwargs.py
+++ b/apps/python-sdk/tests/test_crawl_direct_kwargs.py
@@ -190,3 +190,72 @@ class TestAsyncCrawlDirectKwargs:
assert captured["integration"] == "test-int"
assert captured["scrape_options"] is not None
+
+
+class TestThreatProtectionDirectKwargs:
+ EXPECTED_WIRE_PAYLOAD = {
+ "mode": "normal",
+ "riskScoreThreshold": 80,
+ "blockedTlds": ["zip"],
+ }
+
+ @staticmethod
+ def _threat_protection():
+ from firecrawl.v2.types import ThreatProtectionOptions
+
+ return ThreatProtectionOptions(
+ mode="normal", risk_score_threshold=80, blocked_tlds=["zip"]
+ )
+
+ def test_sync_threat_protection_kwarg_lands_in_wire_body(self, client, monkeypatch):
+ captured = {}
+
+ def fake_start_crawl(http, request):
+ captured["request"] = request
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ client.start_crawl(
+ "https://example.com",
+ threat_protection=self._threat_protection(),
+ )
+
+ request = captured["request"]
+ assert request.scrape_options is not None
+ assert request.scrape_options.threat_protection is not None
+ wire_body = crawl_mod._prepare_crawl_request(request)
+ assert (
+ wire_body["scrapeOptions"]["threatProtection"]
+ == self.EXPECTED_WIRE_PAYLOAD
+ )
+
+ def test_async_threat_protection_kwarg_lands_in_wire_body(self, async_client, monkeypatch):
+ captured = {}
+
+ async def fake_start_crawl(http, request):
+ captured["request"] = request
+ raise ConnectionError("captured")
+
+ monkeypatch.setattr(async_crawl_mod, "start_crawl", fake_start_crawl)
+
+ with pytest.raises(ConnectionError):
+ asyncio.get_event_loop().run_until_complete(
+ async_client.start_crawl(
+ "https://example.com",
+ threat_protection=self._threat_protection(),
+ )
+ )
+
+ request = captured["request"]
+ assert request.scrape_options is not None
+ assert request.scrape_options.threat_protection is not None
+ wire_body = async_crawl_mod._prepare_crawl_request(request)
+ assert (
+ wire_body["scrapeOptions"]["threatProtection"]
+ == self.EXPECTED_WIRE_PAYLOAD
+ )
+
+ def test_threat_protection_in_scrape_option_keys(self):
+ assert "threat_protection" in _SCRAPE_OPTION_KEYS