Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdk): add agent webhook support to Node.js and Python SDKs (#2705)
firecrawl-spring[bot]f4c3fcedadc4
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent.test.tsnew file mode 100644index 000000000..ce9535ee7--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/agent.test.ts@@ -0,0 +1,80 @@+import { describe, test, expect } from "@jest/globals";++// We need to test the prepareAgentPayload function, but it's not exported.+// Since the function is internal, we'll test the behavior through type checking+// and verify the types are properly exported.++import type { AgentWebhookConfig, AgentWebhookEvent } from "../../../v2/types";++describe("v2 types: Agent webhook types", () => {+ test("AgentWebhookConfig accepts string webhook", () => {+ // Type check - this should compile without errors+ const webhook: string | AgentWebhookConfig = "https://example.com/webhook";+ expect(typeof webhook).toBe("string");+ });++ test("AgentWebhookConfig accepts config object", () => {+ const config: AgentWebhookConfig = {+ url: "https://example.com/webhook",+ headers: { Authorization: "Bearer token" },+ events: ["completed", "failed"],+ };+ expect(config.url).toBe("https://example.com/webhook");+ expect(config.headers).toEqual({ Authorization: "Bearer token" });+ expect(config.events).toEqual(["completed", "failed"]);+ });++ test("AgentWebhookConfig accepts minimal config", () => {+ const config: AgentWebhookConfig = {+ url: "https://example.com/webhook",+ };+ expect(config.url).toBe("https://example.com/webhook");+ expect(config.headers).toBeUndefined();+ expect(config.metadata).toBeUndefined();+ expect(config.events).toBeUndefined();+ });++ test("AgentWebhookEvent includes agent-specific events", () => {+ const events: AgentWebhookEvent[] = [+ "started",+ "action",+ "completed",+ "failed",+ "cancelled",+ ];+ expect(events).toContain("action");+ expect(events).toContain("cancelled");+ expect(events.length).toBe(5);+ });++ test("AgentWebhookConfig accepts all fields", () => {+ const config: AgentWebhookConfig = {+ url: "https://example.com/webhook",+ headers: {+ Authorization: "Bearer token",+ "X-Custom-Header": "value",+ },+ metadata: {+ project: "test",+ environment: "staging",+ },+ events: ["started", "action", "completed", "failed", "cancelled"],+ };+ expect(config.url).toBe("https://example.com/webhook");+ expect(Object.keys(config.headers!).length).toBe(2);+ expect(config.metadata!.project).toBe("test");+ expect(config.events!.length).toBe(5);+ });++ test("AgentWebhookConfig events are agent-specific (not crawl)", () => {+ // Agent has 'action' and 'cancelled', but not 'page'+ const config: AgentWebhookConfig = {+ url: "https://example.com/webhook",+ events: ["action", "cancelled"],+ };+ expect(config.events).toContain("action");+ expect(config.events).toContain("cancelled");+ // 'page' is a crawl-specific event, not valid for agent+ // This is enforced at the type level+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts b/apps/js-sdk/firecrawl/src/v2/methods/agent.tsindex af6cd4fca..9200cfce2 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/agent.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/agent.ts@@ -1,4 +1,4 @@-import { type AgentResponse, type AgentStatusResponse } from "../types";+import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";import { HttpClient } from "../utils/httpClient";import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";import { zodToJsonSchema } from "zod-to-json-schema";@@ -12,6 +12,7 @@ function prepareAgentPayload(args: {maxCredits?: number;strictConstrainToURLs?: boolean;model?: "spark-1-pro" | "spark-1-mini";+ webhook?: string | AgentWebhookConfig;}): Record<string, unknown> {const body: Record<string, unknown> = {};if (args.urls) body.urls = args.urls;@@ -25,6 +26,7 @@ function prepareAgentPayload(args: {if (args.maxCredits !== null && args.maxCredits !== undefined) body.maxCredits = args.maxCredits;if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;if (args.model !== null && args.model !== undefined) body.model = args.model;+ if (args.webhook != null) body.webhook = args.webhook;return body;}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 5869be0d6..71e6d4e0a 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -167,6 +167,16 @@ export interface WebhookConfig {events?: Array<'completed' | 'failed' | 'page' | 'started'>;}+// Agent webhook events differ from crawl: has 'action' and 'cancelled', no 'page'+export type AgentWebhookEvent = 'started' | 'action' | 'completed' | 'failed' | 'cancelled';++export interface AgentWebhookConfig {+ url: string;+ headers?: Record<string, string>;+ metadata?: Record<string, string>;+ events?: AgentWebhookEvent[];+}+export interface BrandingProfile {colorScheme?: 'light' | 'dark';logo?: string | null;diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_request_preparation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_request_preparation.pyindex 7cb30d195..6ccc7225d 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_request_preparation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_request_preparation.py@@ -7,6 +7,7 @@ from pydantic import BaseModel, Fieldfrom typing import List, Optionalfrom firecrawl.v2.methods.agent import _prepare_agent_request+from firecrawl.v2.types import AgentWebhookConfigclass TestAgentRequestPreparation:@@ -224,3 +225,115 @@ class TestAgentRequestPreparation:schema=["not", "a", "valid", "schema"])+ def test_request_with_string_webhook(self):+ """Test request preparation with string webhook URL."""+ data = _prepare_agent_request(+ None,+ prompt="Test prompt",+ webhook="https://example.com/webhook"+ )++ assert data["webhook"] == "https://example.com/webhook"++ def test_request_with_webhook_config(self):+ """Test request preparation with AgentWebhookConfig object."""+ webhook_config = AgentWebhookConfig(+ url="https://example.com/webhook",+ headers={"Authorization": "Bearer token"},+ events=["completed", "failed"]+ )+ data = _prepare_agent_request(+ None,+ prompt="Test prompt",+ webhook=webhook_config+ )++ assert data["webhook"]["url"] == "https://example.com/webhook"+ assert data["webhook"]["headers"] == {"Authorization": "Bearer token"}+ assert data["webhook"]["events"] == ["completed", "failed"]++ def test_request_without_webhook(self):+ """Test that webhook is not included when None."""+ data = _prepare_agent_request(+ None,+ prompt="Test prompt"+ )++ assert "webhook" not in data++ def test_webhook_config_excludes_none_values(self):+ """Test that None values are excluded from webhook config."""+ webhook_config = AgentWebhookConfig(+ url="https://example.com/webhook"+ )+ data = _prepare_agent_request(+ None,+ prompt="Test prompt",+ webhook=webhook_config+ )++ assert "headers" not in data["webhook"]+ assert "metadata" not in data["webhook"]+ assert "events" not in data["webhook"]++ def test_agent_specific_webhook_events(self):+ """Test that agent-specific events (action, cancelled) are accepted."""+ webhook_config = AgentWebhookConfig(+ url="https://example.com/webhook",+ events=["started", "action", "completed", "failed", "cancelled"]+ )+ data = _prepare_agent_request(+ None,+ prompt="Test prompt",+ webhook=webhook_config+ )++ assert "action" in data["webhook"]["events"]+ assert "cancelled" in data["webhook"]["events"]++ def test_webhook_with_metadata(self):+ """Test webhook config with metadata."""+ webhook_config = AgentWebhookConfig(+ url="https://example.com/webhook",+ metadata={"project": "test", "env": "staging"}+ )+ data = _prepare_agent_request(+ None,+ prompt="Test prompt",+ webhook=webhook_config+ )++ assert data["webhook"]["metadata"] == {"project": "test", "env": "staging"}++ def test_request_all_fields_with_webhook(self):+ """Test request preparation with all fields including webhook."""+ schema = {+ "type": "object",+ "properties": {"test": {"type": "string"}}+ }+ urls = ["https://example.com"]+ webhook_config = AgentWebhookConfig(+ url="https://example.com/webhook",+ events=["completed"]+ )++ data = _prepare_agent_request(+ urls,+ prompt="Complete test",+ schema=schema,+ integration="test-integration",+ max_credits=50,+ strict_constrain_to_urls=True,+ model="spark-1-pro",+ webhook=webhook_config+ )++ assert data["prompt"] == "Complete test"+ assert data["urls"] == urls+ assert data["schema"] == schema+ assert data["integration"] == "test-integration"+ assert data["maxCredits"] == 50+ assert data["strictConstrainToURLs"] is True+ assert data["model"] == "spark-1-pro"+ assert data["webhook"]["url"] == "https://example.com/webhook"+diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_webhook.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_webhook.pynew file mode 100644index 000000000..b86b3e0c4--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_webhook.py@@ -0,0 +1,92 @@+"""Unit tests for AgentWebhookConfig."""++import pytest+from firecrawl.v2.types import AgentWebhookConfig+++class TestAgentWebhookConfig:+ """Test AgentWebhookConfig class functionality."""++ def test_minimal_config(self):+ """Test creating config with only URL."""+ config = AgentWebhookConfig(url="https://example.com/webhook")+ assert config.url == "https://example.com/webhook"+ assert config.headers is None+ assert config.metadata is None+ assert config.events is None++ def test_full_config(self):+ """Test creating config with all fields."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ headers={"Authorization": "Bearer token"},+ metadata={"project": "test"},+ events=["started", "action", "completed"]+ )+ assert config.url == "https://example.com/webhook"+ assert config.headers == {"Authorization": "Bearer token"}+ assert config.metadata == {"project": "test"}+ assert config.events == ["started", "action", "completed"]++ def test_url_required(self):+ """Test that URL is required."""+ with pytest.raises(Exception):+ AgentWebhookConfig()++ def test_serialization_excludes_none(self):+ """Test model_dump excludes None values."""+ config = AgentWebhookConfig(url="https://example.com/webhook")+ data = config.model_dump(exclude_none=True)+ assert data == {"url": "https://example.com/webhook"}++ def test_serialization_includes_all_fields(self):+ """Test model_dump includes all non-None fields."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ headers={"X-Custom": "value"},+ events=["completed", "failed"]+ )+ data = config.model_dump(exclude_none=True)+ assert data == {+ "url": "https://example.com/webhook",+ "headers": {"X-Custom": "value"},+ "events": ["completed", "failed"]+ }++ def test_agent_specific_events(self):+ """Test that agent-specific events are valid."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ events=["started", "action", "completed", "failed", "cancelled"]+ )+ assert "action" in config.events+ assert "cancelled" in config.events++ def test_single_event(self):+ """Test config with single event."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ events=["completed"]+ )+ assert config.events == ["completed"]++ def test_empty_headers(self):+ """Test config with empty headers dict."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ headers={}+ )+ assert config.headers == {}++ def test_multiple_headers(self):+ """Test config with multiple headers."""+ config = AgentWebhookConfig(+ url="https://example.com/webhook",+ headers={+ "Authorization": "Bearer token",+ "X-Custom-Header": "value",+ "Content-Type": "application/json"+ }+ )+ assert len(config.headers) == 3+ assert config.headers["Authorization"] == "Bearer token"diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 338a61b32..71e5b0bdd 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -21,6 +21,7 @@ from .types import (PDFParser,CrawlParamsData,WebhookConfig,+ AgentWebhookConfig,CrawlErrorsResponse,ActiveCrawlsResponse,MapOptions,@@ -765,6 +766,7 @@ class FirecrawlClient:max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,):"""Start an agent job (non-blocking).@@ -775,6 +777,7 @@ class FirecrawlClient:integration: Integration tag/namemax_credits: Maximum credits to use (optional)model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")+ webhook: Webhook URL or configuration for notificationsReturns:Response payload with job id/status (poll with get_agent_status)"""@@ -787,6 +790,7 @@ class FirecrawlClient:max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)def agent(@@ -801,6 +805,7 @@ class FirecrawlClient:max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,):"""Run an agent and wait until completion.@@ -813,6 +818,7 @@ class FirecrawlClient:timeout: Maximum seconds to wait (None for no timeout)max_credits: Maximum credits to use (optional)model: Model to use for the agent ("spark-1-pro" or "spark-1-mini")+ webhook: Webhook URL or configuration for notificationsReturns:Final agent response when completed"""@@ -827,6 +833,7 @@ class FirecrawlClient:max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)def get_agent_status(self, job_id: str):diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 320a9ab96..ee20f34e3 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -10,6 +10,7 @@ from .types import (ScrapeOptions,CrawlRequest,WebhookConfig,+ AgentWebhookConfig,SearchRequest,SearchData,SourceOption,@@ -323,6 +324,8 @@ class AsyncFirecrawlClient:timeout: Optional[int] = None,max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,+ model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,):return await async_agent.agent(self.async_http_client,@@ -334,6 +337,8 @@ class AsyncFirecrawlClient:timeout=timeout,max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,+ model=model,+ webhook=webhook,)async def get_agent_status(self, job_id: str):@@ -348,6 +353,8 @@ class AsyncFirecrawlClient:integration: Optional[str] = None,max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,+ model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,):return await async_agent.start_agent(self.async_http_client,@@ -357,6 +364,8 @@ class AsyncFirecrawlClient:integration=integration,max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,+ model=model,+ webhook=webhook,)async def cancel_agent(self, job_id: str) -> bool:diff --git a/apps/python-sdk/firecrawl/v2/methods/agent.py b/apps/python-sdk/firecrawl/v2/methods/agent.pyindex a83250c3d..00169c4e9 100644--- a/apps/python-sdk/firecrawl/v2/methods/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/agent.py@@ -1,7 +1,7 @@-from typing import Any, Dict, List, Literal, Optional+from typing import Any, Dict, List, Literal, Optional, Unionimport time-from ..types import AgentResponse+from ..types import AgentResponse, AgentWebhookConfigfrom ..utils.http_client import HttpClientfrom ..utils.error_handler import handle_response_errorfrom ..utils.validation import _normalize_schema@@ -16,6 +16,7 @@ def _prepare_agent_request(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -38,6 +39,11 @@ def _prepare_agent_request(body["strictConstrainToURLs"] = strict_constrain_to_urlsif model is not None:body["model"] = model+ if webhook is not None:+ if isinstance(webhook, str):+ body["webhook"] = webhook+ else:+ body["webhook"] = webhook.model_dump(exclude_none=True)return body@@ -60,6 +66,7 @@ def start_agent(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -69,6 +76,7 @@ def start_agent(max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)resp = client.post("/v2/agent", body)if not resp.ok:@@ -114,6 +122,7 @@ def agent(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> AgentResponse:started = start_agent(client,@@ -124,6 +133,7 @@ def agent(max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.pyindex 87940242f..6190f1f18 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py@@ -1,7 +1,7 @@-from typing import Any, Dict, List, Literal, Optional+from typing import Any, Dict, List, Literal, Optional, Unionimport asyncio-from ...types import AgentResponse+from ...types import AgentResponse, AgentWebhookConfigfrom ...utils.http_client_async import AsyncHttpClientfrom ...utils.validation import _normalize_schema@@ -15,6 +15,7 @@ def _prepare_agent_request(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> Dict[str, Any]:body: Dict[str, Any] = {}if urls is not None:@@ -37,6 +38,11 @@ def _prepare_agent_request(body["strictConstrainToURLs"] = strict_constrain_to_urlsif model is not None:body["model"] = model+ if webhook is not None:+ if isinstance(webhook, str):+ body["webhook"] = webhook+ else:+ body["webhook"] = webhook.model_dump(exclude_none=True)return body@@ -59,6 +65,7 @@ async def start_agent(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> AgentResponse:body = _prepare_agent_request(urls,@@ -68,6 +75,7 @@ async def start_agent(max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)resp = await client.post("/v2/agent", body)payload = _normalize_agent_response_payload(resp.json())@@ -109,6 +117,7 @@ async def agent(max_credits: Optional[int] = None,strict_constrain_to_urls: Optional[bool] = None,model: Optional[Literal["spark-1-pro", "spark-1-mini"]] = None,+ webhook: Optional[Union[str, AgentWebhookConfig]] = None,) -> AgentResponse:started = await start_agent(client,@@ -119,6 +128,7 @@ async def agent(max_credits=max_credits,strict_constrain_to_urls=strict_constrain_to_urls,model=model,+ webhook=webhook,)job_id = getattr(started, "id", None)if not job_id:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 5b75c892a..673870939 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -320,6 +320,23 @@ class WebhookConfig(BaseModel):events: Optional[List[Literal["completed", "failed", "page", "started"]]] = None+class AgentWebhookConfig(BaseModel):+ """Configuration for agent webhooks.++ Agent webhooks support different events than crawl webhooks:+ - started: When the agent job starts+ - action: When the agent takes an action/step+ - completed: When the job completes successfully+ - failed: When the job fails+ - cancelled: When the job is cancelled+ """++ url: str+ headers: Optional[Dict[str, str]] = None+ metadata: Optional[Dict[str, str]] = None+ events: Optional[List[Literal["started", "action", "completed", "failed", "cancelled"]]] = None++class WebhookData(BaseModel):"""Data sent to webhooks."""