Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(python-sdk): configurable origin attribution for Arcade and other integrators (#4440)
Erik Engervall1715e4cca81d
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_hints.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_hints.pyindex 87288d46d..da6c28ae6 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_hints.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_agent_hints.py@@ -30,6 +30,7 @@ def body():@pytest.mark.parametrize("operation", ["search", "scrape", "parse", "map"])def test_sync_hints_survive_unwrapping(operation):client = Mock()+ client.origin = "python-sdk@test"client.post.return_value = client.post_multipart.return_value = response(body())calls = {"search": lambda: search(client, SearchRequest(query="test")),@@ -48,6 +49,7 @@ def test_sync_hints_survive_unwrapping(operation):@pytest.mark.parametrize("operation", ["search", "scrape", "parse", "map"])async def test_async_hints_survive_unwrapping(operation):client = Mock()+ client.origin = "python-sdk@test"client.post = AsyncMock(return_value=response(body()))client.post_multipart = AsyncMock(return_value=response(body()))calls = {diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.pynew file mode 100644index 000000000..971b19e90--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_origin.py@@ -0,0 +1,146 @@+"""Unit tests for configurable request origin attribution."""++from firecrawl.v2.utils.http_client import HttpClient+from firecrawl.v2.utils.http_client_async import AsyncHttpClient+from firecrawl.v2.utils.get_version import get_version+++def test_http_client_default_origin():+ client = HttpClient(api_key="fc-test", api_url="https://api.firecrawl.dev")+ assert client.origin == f"python-sdk@{get_version()}"+++def test_http_client_custom_origin():+ client = HttpClient(+ api_key="fc-test",+ api_url="https://api.firecrawl.dev",+ origin="arcade-mcp",+ )+ assert client.origin == "arcade-mcp"+++def test_async_http_client_custom_origin():+ client = AsyncHttpClient(+ api_key="fc-test",+ api_url="https://api.firecrawl.dev",+ origin="arcade-mcp",+ )+ assert client.origin == "arcade-mcp"+++def test_firecrawl_clients_construct_with_origin_and_expose_wait_crawl():+ from firecrawl import Firecrawl, AsyncFirecrawl++ sync_client = Firecrawl(api_key="fc-test", origin="arcade-mcp")+ assert sync_client._v2_client.http_client.origin == "arcade-mcp"+ assert callable(sync_client.wait_crawl)++ async_client = AsyncFirecrawl(api_key="fc-test", origin="arcade-mcp")+ assert callable(async_client.wait_crawl)+++def test_sync_wait_crawl_polls_until_terminal(monkeypatch):+ from firecrawl.v2.client import FirecrawlClient+ from firecrawl.v2.methods import crawl as crawl_module+ from firecrawl.v2.types import CrawlJob++ statuses = iter(["scraping", "completed"])++ def fake_status(client, job_id, request_timeout=None):+ return CrawlJob(status=next(statuses), completed=0, total=0, credits_used=0, data=[])++ monkeypatch.setattr(crawl_module, "get_crawl_status", fake_status)+ monkeypatch.setattr(crawl_module.time, "sleep", lambda _: None)++ client = FirecrawlClient(api_key="fc-test")+ assert client.wait_crawl("job-1", poll_interval=0).status == "completed"+++class _FakeResponse:+ status_code = 200+ ok = True++ def json(self):+ return {"success": True, "data": {}}+++def _capture_sync(monkeypatch, method):+ import firecrawl.v2.utils.http_client as http_module++ sent = {}++ def fake_request(url, headers=None, json=None, timeout=None, **kwargs):+ sent["json"] = json+ return _FakeResponse()++ monkeypatch.setattr(http_module.requests, method, fake_request)+ return sent+++def test_sync_post_and_patch_stamp_client_origin(monkeypatch):+ client = HttpClient(api_key="fc-test", api_url="https://api.firecrawl.dev", origin="arcade-mcp")++ sent = _capture_sync(monkeypatch, "post")+ client.post("/v2/scrape", {"url": "https://example.com"})+ assert sent["json"]["origin"] == "arcade-mcp"++ sent = _capture_sync(monkeypatch, "patch")+ client.patch("/v2/monitor/x", {"name": "n"})+ assert sent["json"]["origin"] == "arcade-mcp"+++def test_sync_post_keeps_explicit_request_origin(monkeypatch):+ client = HttpClient(api_key="fc-test", api_url="https://api.firecrawl.dev", origin="arcade-mcp")+ sent = _capture_sync(monkeypatch, "post")+ client.post("/v2/scrape", {"url": "https://example.com", "origin": "per-call"})+ assert sent["json"]["origin"] == "per-call"+++def test_async_post_stamps_client_origin_and_keeps_explicit_one():+ import asyncio+ import json+ import httpx++ sent = []++ def handler(request):+ sent.append(json.loads(request.content))+ return httpx.Response(200, json={"success": True})++ async def run():+ client = AsyncHttpClient(api_key="fc-test", api_url="https://api.firecrawl.dev", origin="arcade-mcp")+ client._client = httpx.AsyncClient(+ base_url="https://api.firecrawl.dev", transport=httpx.MockTransport(handler)+ )+ await client.post("/v2/scrape", {"url": "https://example.com"})+ await client.post("/v2/scrape", {"url": "https://example.com", "origin": "per-call"})+ await client.close()++ asyncio.run(run())+ assert [body["origin"] for body in sent] == ["arcade-mcp", "per-call"]+++def test_parse_multipart_options_carry_client_origin():+ import json+ from firecrawl.v2.methods.parse import _prepare_parse_request++ fields, _ = _prepare_parse_request(b"hello", filename="a.txt", origin="arcade-mcp")+ assert json.loads(fields["options"])["origin"] == "arcade-mcp"++ fields, _ = _prepare_parse_request(b"hello", filename="a.txt")+ assert json.loads(fields["options"])["origin"] == f"python-sdk@{get_version()}"+++def test_research_get_query_carries_client_origin():+ from firecrawl.v2.methods import research++ class FakeClient:+ origin = "arcade-mcp"++ def get(self, path):+ self.path = path+ return _FakeResponse()++ client = FakeClient()+ research.search_papers(client, "transformers")+ assert "origin=arcade-mcp" in client.pathdiff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 2a64d72aa..b319a4db9 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -247,6 +247,7 @@ class Firecrawl:timeout: float = None,max_retries: int = 3,backoff_factor: float = 0.5,+ origin: str = None,):"""Initialize the unified client.@@ -256,6 +257,8 @@ class Firecrawl:timeout: Default request timeout in seconds for all HTTP requestsmax_retries: Maximum number of retries for failed requests (default: 3)backoff_factor: Exponential backoff factor for retries (default: 0.5)+ origin: Attribution string stamped into API request payloads+ (defaults to ``python-sdk@<version>``)"""self.api_key = api_keyself.api_url = api_url@@ -268,6 +271,7 @@ class Firecrawl:timeout=timeout,max_retries=max_retries,backoff_factor=backoff_factor,+ origin=origin,) if V2FirecrawlClient else None# Create version-specific proxies@@ -297,6 +301,7 @@ class Firecrawl:self.crawl = self._v2_client.crawlself.start_crawl = self._v2_client.start_crawl+ self.wait_crawl = self._v2_client.wait_crawlself.crawl_params_preview = self._v2_client.crawl_params_previewself.get_crawl_status = self._v2_client.get_crawl_statusself.get_crawl_status_page = self._v2_client.get_crawl_status_page@@ -395,6 +400,7 @@ class AsyncFirecrawl:timeout: float = None,max_retries: int = 3,backoff_factor: float = 0.5,+ origin: str = None,):self.api_key = api_keyself.api_url = api_url@@ -407,6 +413,7 @@ class AsyncFirecrawl:timeout=timeout,max_retries=max_retries,backoff_factor=backoff_factor,+ origin=origin,) if AsyncFirecrawlClient else None# Create version-specific proxies@@ -437,6 +444,7 @@ class AsyncFirecrawl:self.get_monitor_check = self._v2_client.get_monitor_checkself.start_crawl = self._v2_client.start_crawl+ self.wait_crawl = self._v2_client.wait_crawlself.get_crawl_status = self._v2_client.get_crawl_statusself.get_crawl_status_page = self._v2_client.get_crawl_status_pageself.cancel_crawl = self._v2_client.cancel_crawldiff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex b95f844e7..dc0b09ac3 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -116,7 +116,8 @@ class FirecrawlClient:api_url: str = "https://api.firecrawl.dev",timeout: Optional[float] = None,max_retries: int = 3,- backoff_factor: float = 0.5+ backoff_factor: float = 0.5,+ origin: Optional[str] = None,):"""Initialize the Firecrawl client.@@ -127,6 +128,8 @@ class FirecrawlClient:timeout: Request timeout in secondsmax_retries: Maximum number of retries for failed requestsbackoff_factor: Exponential backoff factor for retries (e.g. 0.5 means wait 0.5s, then 1s, then 2s between retries)+ origin: Attribution string stamped into API request payloads+ (defaults to ``python-sdk@<version>``)"""if api_key is None:api_key = os.getenv("FIRECRAWL_API_KEY")@@ -150,6 +153,7 @@ class FirecrawlClient:timeout=timeout,max_retries=max_retries,backoff_factor=backoff_factor,+ origin=origin,)def scrape(@@ -881,7 +885,38 @@ class FirecrawlClient:request = CrawlRequest(**request_kwargs)return crawl_module.start_crawl(self.http_client, request)-++ def wait_crawl(+ self,+ job_id: str,+ poll_interval: int = 2,+ timeout: Optional[int] = None,+ *,+ request_timeout: Optional[float] = None,+ ) -> CrawlJob:+ """+ Poll a crawl job until it reaches a terminal state.++ Args:+ job_id: ID of the crawl job+ poll_interval: Seconds between status checks+ timeout: Maximum seconds to wait for the whole job (None waits indefinitely)+ request_timeout: Optional timeout (in seconds) for each status request++ Returns:+ CrawlJob in a terminal state ("completed", "failed", or "cancelled")++ Raises:+ TimeoutError: If the job does not finish within timeout+ """+ return crawl_module.wait_for_crawl_completion(+ self.http_client,+ job_id,+ poll_interval=poll_interval,+ timeout=timeout,+ request_timeout=request_timeout,+ )+def get_crawl_status(self,job_id: str,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 0b745e087..1307a2117 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -94,6 +94,7 @@ class AsyncFirecrawlClient:timeout: Optional[float] = None,max_retries: int = 3,backoff_factor: float = 0.5,+ origin: Optional[str] = None,):if api_key is None:api_key = os.getenv("FIRECRAWL_API_KEY")@@ -105,6 +106,7 @@ class AsyncFirecrawlClient:timeout=timeout,max_retries=max_retries,backoff_factor=backoff_factor,+ origin=origin,)self.async_http_client = AsyncHttpClient(api_key,@@ -112,6 +114,7 @@ class AsyncFirecrawlClient:timeout=timeout,max_retries=max_retries,backoff_factor=backoff_factor,+ origin=origin,)# Scrapediff --git a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py b/apps/python-sdk/firecrawl/v2/methods/aio/agent.pyindex 023d5578f..d4ce238cb 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/agent.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/agent.py@@ -125,7 +125,7 @@ async def start_agent(exchange=exchange,)resp = await client.post("/v2/agent", body)- if not resp.ok:+ if resp.status_code >= 400:handle_response_error(resp, "agent")payload = _normalize_agent_response_payload(resp.json())return AgentResponse(**payload)@@ -133,7 +133,7 @@ async def start_agent(async def get_agent_status(client: AsyncHttpClient, job_id: str) -> AgentResponse:resp = await client.get(f"/v2/agent/{job_id}")- if not resp.ok:+ if resp.status_code >= 400:handle_response_error(resp, "agent-status")payload = _normalize_agent_response_payload(resp.json())return AgentResponse(**payload)@@ -243,7 +243,7 @@ async def get_agent_trace(if live_view:endpoint += "?liveView=true"resp = await client.get(endpoint)- if not resp.ok:+ if resp.status_code >= 400:handle_response_error(resp, "agent-trace")return AgentTraceResponse(**resp.json())@@ -283,7 +283,7 @@ async def get_agent_snapshot(snapshot_id: Snapshot ID from an artifact.updated trace event"""resp = await client.get(f"/v2/agent/{job_id}/snapshots/{snapshot_id}")- if not resp.ok:+ if resp.status_code >= 400:handle_response_error(resp, "agent-snapshot")return AgentSnapshotResponse(**resp.json())@@ -303,6 +303,6 @@ async def cancel_agent(client: AsyncHttpClient, job_id: str) -> bool:Exception: If the cancellation fails"""resp = await client.delete(f"/v2/agent/{job_id}")- if not resp.ok:+ if resp.status_code >= 400:handle_response_error(resp, "cancel agent")return resp.json().get("success", False)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/parse.py b/apps/python-sdk/firecrawl/v2/methods/aio/parse.pyindex 9a599b6e8..0faf86b9c 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/parse.py@@ -20,8 +20,9 @@ async def _prepare_parse_request(*,filename: Optional[str] = None,content_type: Optional[str] = None,+ origin: Optional[str] = None,) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:- request_data = _prepare_parse_options_payload(options)+ request_data = _prepare_parse_options_payload(options, origin)multipart_fields = {"options": json.dumps(request_data)}loop = asyncio.get_running_loop()multipart_files = await loop.run_in_executor(@@ -49,6 +50,7 @@ async def parse(options,filename=filename,content_type=content_type,+ origin=client.origin,)response = await client.post_multipart("/v2/parse", data=fields, files=files)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/research.py b/apps/python-sdk/firecrawl/v2/methods/aio/research.pyindex 33980e7be..f71cd1265 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/research.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/research.py@@ -46,7 +46,11 @@ from ..research_docs import (BASE = "/v2/search/research"-ORIGIN = f"python-sdk@{get_version()}"+_DEFAULT_ORIGIN = f"python-sdk@{get_version()}"+++def _origin(client: AsyncHttpClient) -> str:+ return getattr(client, "origin", None) or _DEFAULT_ORIGINdef _query(params: Dict[str, Any]) -> str:@@ -91,7 +95,7 @@ async def search_papers("categories": categories,"from": from_date,"to": to_date,- "origin": ORIGIN,+ "origin": _origin(client),}),)@@ -101,7 +105,7 @@ async def search_papers(async def inspect_paper(client: AsyncHttpClient, paper_id: str) -> Dict[str, Any]:return await _get(client,- f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": _origin(client)}),)@@ -116,7 +120,7 @@ async def read_paper(return await _get(client,f"{BASE}/papers/{quote(paper_id, safe='')}"- + _query({"query": query, "k": k, "origin": ORIGIN}),+ + _query({"query": query, "k": k, "origin": _origin(client)}),)@@ -141,7 +145,7 @@ async def related_papers("k": k,"rerank": None if rerank is None else str(rerank).lower(),"anchor": anchor,- "origin": ORIGIN,+ "origin": _origin(client),}),)@@ -159,5 +163,5 @@ async def search_github(warnings.warn(GITHUB_SEARCH_DEPRECATION_MSG, FutureWarning, stacklevel=2)return await _get(client,- BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}),+ BASE + "/github" + _query({"query": query, "k": k, "origin": _origin(client)}),)diff --git a/apps/python-sdk/firecrawl/v2/methods/parse.py b/apps/python-sdk/firecrawl/v2/methods/parse.pyindex 22e43fbaa..eb13fc530 100644--- a/apps/python-sdk/firecrawl/v2/methods/parse.py+++ b/apps/python-sdk/firecrawl/v2/methods/parse.py@@ -61,6 +61,7 @@ def _validate_parse_options_payload(options_payload: Dict[str, Any]) -> None:def _prepare_parse_options_payload(options: Optional[ParseOptions],+ origin: Optional[str] = None,) -> Dict[str, Any]:request_data: Dict[str, Any] = {}@@ -76,7 +77,7 @@ def _prepare_parse_options_payload(opts.pop("lockdown", None)request_data.update(opts)- request_data["origin"] = request_data.get("origin") or f"python-sdk@{version}"+ request_data["origin"] = request_data.get("origin") or origin or f"python-sdk@{version}"return request_data@@ -126,8 +127,9 @@ def _prepare_parse_request(*,filename: Optional[str] = None,content_type: Optional[str] = None,+ origin: Optional[str] = None,) -> Tuple[Dict[str, Any], Dict[str, Tuple[str, bytes, str]]]:- request_data = _prepare_parse_options_payload(options)+ request_data = _prepare_parse_options_payload(options, origin)multipart_fields = {"options": json.dumps(request_data)}multipart_files = _prepare_file_payload(file,@@ -150,6 +152,7 @@ def parse(options,filename=filename,content_type=content_type,+ origin=client.origin,)response = client.post_multipart("/v2/parse", data=fields, files=files)diff --git a/apps/python-sdk/firecrawl/v2/methods/research.py b/apps/python-sdk/firecrawl/v2/methods/research.pyindex 6bf51b8b4..f44eb012b 100644--- a/apps/python-sdk/firecrawl/v2/methods/research.py+++ b/apps/python-sdk/firecrawl/v2/methods/research.py@@ -45,7 +45,11 @@ from .research_docs import (BASE = "/v2/search/research"-ORIGIN = f"python-sdk@{get_version()}"+_DEFAULT_ORIGIN = f"python-sdk@{get_version()}"+++def _origin(client: HttpClient) -> str:+ return getattr(client, "origin", None) or _DEFAULT_ORIGINdef _query(params: Dict[str, Any]) -> str:@@ -90,7 +94,7 @@ def search_papers("categories": categories,"from": from_date,"to": to_date,- "origin": ORIGIN,+ "origin": _origin(client),}),)@@ -100,7 +104,7 @@ def search_papers(def inspect_paper(client: HttpClient, paper_id: str) -> Dict[str, Any]:return _get(client,- f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": _origin(client)}),)@@ -115,7 +119,7 @@ def read_paper(return _get(client,f"{BASE}/papers/{quote(paper_id, safe='')}"- + _query({"query": query, "k": k, "origin": ORIGIN}),+ + _query({"query": query, "k": k, "origin": _origin(client)}),)@@ -140,7 +144,7 @@ def related_papers("k": k,"rerank": None if rerank is None else str(rerank).lower(),"anchor": anchor,- "origin": ORIGIN,+ "origin": _origin(client),}),)@@ -158,5 +162,5 @@ def search_github(warnings.warn(GITHUB_SEARCH_DEPRECATION_MSG, FutureWarning, stacklevel=2)return _get(client,- BASE + "/github" + _query({"query": query, "k": k, "origin": ORIGIN}),+ BASE + "/github" + _query({"query": query, "k": k, "origin": _origin(client)}),)diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pyindex 4a780cc19..20fd4d2a2 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -21,12 +21,15 @@ class HttpClient:timeout: Optional[float] = None,max_retries: int = 3,backoff_factor: float = 0.5,+ origin: Optional[str] = None,):self.api_key = api_keyself.api_url = api_urlself.timeout = timeoutself.max_retries = max_retriesself.backoff_factor = backoff_factor+ # Attribution string stamped into request payloads / research query params.+ self.origin = origin or f"python-sdk@{version}"def _build_url(self, endpoint: str) -> str:if urlparse(endpoint).netloc:@@ -73,7 +76,7 @@ class HttpClient:backoff_factor = self.backoff_factorpayload = dict(data)- payload['origin'] = f'python-sdk@{version}'+ payload['origin'] = payload.get('origin') or self.originurl = self._build_url(endpoint)@@ -270,7 +273,7 @@ class HttpClient:backoff_factor = self.backoff_factorpayload = dict(data)- payload['origin'] = f'python-sdk@{version}'+ payload['origin'] = payload.get('origin') or self.originurl = self._build_url(endpoint)last_exception = Nonediff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex b1d270b5e..4f08b13f9 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -15,12 +15,15 @@ class AsyncHttpClient:timeout: Optional[float] = None,max_retries: int = 3,backoff_factor: float = 0.5,+ origin: Optional[str] = None,):self.api_key = api_keyself.api_url = api_urlself.timeout = timeoutself.max_retries = max_retriesself.backoff_factor = backoff_factor+ # Attribution string stamped into request payloads / research query params.+ self.origin = origin or f"python-sdk@{version}"headers = {}@@ -62,7 +65,7 @@ class AsyncHttpClient:backoff_factor = self.backoff_factorpayload = dict(data)- payload["origin"] = f"python-sdk@{version}"+ payload["origin"] = payload.get("origin") or self.originlast_exception = Nonenum_attempts = max(1, retries)@@ -227,7 +230,7 @@ class AsyncHttpClient:backoff_factor = self.backoff_factorpayload = dict(data)- payload["origin"] = f"python-sdk@{version}"+ payload["origin"] = payload.get("origin") or self.originlast_exception = Nonenum_attempts = max(1, retries)