Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(python-sdk): stop sending the API key to cross-host next URLs (#4861)
Gergő Móricz70a134c4df8d
diff --git a/.github/workflows/test-python-sdk.yml b/.github/workflows/test-python-sdk.ymlnew file mode 100644index 000000000..bbbfcc454--- /dev/null+++ b/.github/workflows/test-python-sdk.yml@@ -0,0 +1,39 @@+name: Python SDK Test Suite++on:+ pull_request:+ branches:+ - main+ paths:+ - apps/python-sdk/**+ - .github/workflows/test-python-sdk.yml++permissions:+ contents: read++jobs:+ unit:+ name: Unit tests+ runs-on: blacksmith-2vcpu-ubuntu-2404+ steps:+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0+ with:+ persist-credentials: false++ - name: Set up Python+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0+ with:+ python-version: "3.12"+ cache: pip+ cache-dependency-path: ./apps/python-sdk/requirements.txt++ - name: Install dependencies+ working-directory: ./apps/python-sdk+ run: |+ python -m pip install --upgrade pip+ pip install -r requirements.txt+ pip install -e .++ - name: Run unit tests+ working-directory: ./apps/python-sdk+ run: python -m pytest firecrawl/__tests__/unit -qdiff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 459c68192..6ebce505e 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -27,7 +27,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.45.0"+__version__ = "4.45.1"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.py b/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.pynew file mode 100644index 000000000..a32698cfa--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.py@@ -0,0 +1,77 @@+"""The v1 clients must never follow a `next` URL off the configured api_url origin."""++import asyncio+from unittest.mock import AsyncMock, MagicMock, patch++import pytest++from firecrawl.v1.client import AsyncV1FirecrawlApp, V1FirecrawlApp++API_URL = "https://api.firecrawl.dev"++NEXT_URLS = [+ ("https://evil.example.com/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),+ ("//evil.example.com/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),+ ("http://api.firecrawl.dev:8443/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),+ ("https://evil.example.com/v1/x/id;p=1?skip=1", f"{API_URL}/v1/x/id;p=1?skip=1"),+ (f"{API_URL}/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),+]++SYNC_CALLS = [+ ("check_crawl_status", lambda app: app.check_crawl_status("id"), "/v1/crawl/id"),+ ("check_batch_scrape_status", lambda app: app.check_batch_scrape_status("id"), "/v1/batch/scrape/id"),+ ("_monitor_job_status", lambda app: app._monitor_job_status("id", {}, 2), "/v1/crawl/id"),+]++ASYNC_CALLS = [+ ("check_crawl_status", lambda app: app.check_crawl_status("id"), "/v1/crawl/id"),+ ("check_batch_scrape_status", lambda app: app.check_batch_scrape_status("id"), "/v1/batch/scrape/id"),+ ("_async_monitor_job_status", lambda app: app._async_monitor_job_status("id", {}), "/v1/crawl/id"),+]+++def _page(markdown, next_url=None):+ page = {"success": True, "status": "completed", "completed": 2, "total": 2,+ "creditsUsed": 2, "expiresAt": "2026-10-01T00:00:00Z",+ "data": [{"markdown": markdown}]}+ if next_url:+ page["next"] = next_url+ return page++[email protected]("name, call, first_path", SYNC_CALLS, ids=[c[0] for c in SYNC_CALLS])[email protected]("next_url, expected", NEXT_URLS)+def test_sync_pagination_pins_next_url(name, call, first_path, next_url, expected):+ app = V1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)+ responses = [+ MagicMock(status_code=200, json=MagicMock(return_value=_page("a", next_url))),+ MagicMock(status_code=200, json=MagicMock(return_value=_page("b"))),+ ]+ with patch("firecrawl.v1.client.requests.get", side_effect=responses) as get:+ call(app)++ assert [c.args[0] for c in get.call_args_list] == [f"{API_URL}{first_path}", expected]++[email protected]("name, call, first_path", ASYNC_CALLS, ids=[c[0] for c in ASYNC_CALLS])[email protected]("next_url, expected", NEXT_URLS)+def test_async_pagination_pins_next_url(name, call, first_path, next_url, expected):+ app = AsyncV1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)+ app._async_get_request = AsyncMock(side_effect=[_page("a", next_url), _page("b")])++ asyncio.run(call(app))++ urls = [c.args[0] for c in app._async_get_request.await_args_list]+ assert urls == [f"{API_URL}{first_path}", expected]+++def test_sync_crawl_status_keeps_all_pages():+ app = V1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)+ responses = [+ MagicMock(status_code=200, json=MagicMock(return_value=_page("a", "https://evil.example.com/v1/crawl/id?skip=1"))),+ MagicMock(status_code=200, json=MagicMock(return_value=_page("b"))),+ ]+ with patch("firecrawl.v1.client.requests.get", side_effect=responses):+ status = app.check_crawl_status("id")++ assert [d.markdown for d in status.data] == ["a", "b"]diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.pynew file mode 100644index 000000000..453b80d8d--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.py@@ -0,0 +1,125 @@+"""The async client must force cross-host absolute URLs back onto api_url.++Mirrors the sync HttpClient._build_url behavior: a `next` URL returned by the+API (or a protocol-relative endpoint) must never be followed to another host,+where the Authorization header would leak the API key.+"""++import asyncio+from unittest.mock import AsyncMock, MagicMock++import pytest++from firecrawl.v2.utils.http_client_async import AsyncHttpClient+++def _client() -> AsyncHttpClient:+ client = AsyncHttpClient(+ api_key="fc-test-key", api_url="https://api.firecrawl.dev"+ )+ client._client = MagicMock()+ response = MagicMock(status_code=200)+ client._client.get = AsyncMock(return_value=response)+ client._client.post = AsyncMock(return_value=response)+ return client+++def test_get_rewrites_cross_host_next_url():+ client = _client()+ asyncio.run(+ client.get("https://evil.example.com/v2/team/crawl/id?cursor=abc")+ )+ url = client._client.get.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id?cursor=abc"+++def test_get_rewrites_protocol_relative_cross_host_url():+ client = _client()+ asyncio.run(client.get("//evil.example.com/v2/team/crawl/id"))+ url = client._client.get.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"+++def test_get_keeps_relative_endpoint_untouched():+ client = _client()+ asyncio.run(client.get("/v2/team/crawl/id"))+ url = client._client.get.await_args.args[0]+ assert url == "/v2/team/crawl/id"+++def test_get_rewrites_same_host_alternate_port():+ client = _client()+ asyncio.run(client.get("https://api.firecrawl.dev:8443/v2/team/crawl/id"))+ url = client._client.get.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"+++def test_get_rewrites_same_host_http_url():+ client = _client()+ asyncio.run(client.get("http://api.firecrawl.dev/v2/team/crawl/id"))+ url = client._client.get.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"+++def test_post_rewrites_cross_host_url():+ client = _client()+ asyncio.run(+ client.post(+ "https://evil.example.com/v2/scrape", data={"url": "https://x.test"}+ )+ )+ url = client._client.post.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/scrape"+++ "verb, call",+ [+ ("get", lambda c, url: c.get(url)),+ ("post", lambda c, url: c.post(url, data={})),+ ("post", lambda c, url: c.post_multipart(url, data={}, files={})),+ ("delete", lambda c, url: c.delete(url)),+ ("patch", lambda c, url: c.patch(url, data={})),+ ],+)+def test_every_verb_rewrites_cross_host_url(verb, call):+ client = _client()+ setattr(client._client, verb, AsyncMock(return_value=MagicMock(status_code=200)))+ asyncio.run(call(client, "https://evil.example.com/v2/team/crawl/id?x=1"))+ url = getattr(client._client, verb).await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id?x=1"+++def test_auto_paginated_crawl_status_never_leaves_api_host():+ from firecrawl.v2.methods.aio.crawl import get_crawl_status++ client = _client()+ pages = [+ {"success": True, "status": "completed", "completed": 2, "total": 2,+ "creditsUsed": 2, "data": [{"markdown": "a"}],+ "next": "https://evil.example.com/v2/crawl/id?skip=1"},+ {"success": True, "status": "completed", "completed": 2, "total": 2,+ "creditsUsed": 2, "data": [{"markdown": "b"}]},+ ]+ client._client.get = AsyncMock(+ side_effect=[MagicMock(status_code=200, json=MagicMock(return_value=p)) for p in pages]+ )++ job = asyncio.run(get_crawl_status(client, "id"))++ urls = [c.args[0] for c in client._client.get.await_args_list]+ assert urls == ["/v2/crawl/id", "https://api.firecrawl.dev/v2/crawl/id?skip=1"]+ assert [d.markdown for d in job.data] == ["a", "b"]+++def test_rewrite_keeps_path_params():+ client = _client()+ asyncio.run(client.get("https://evil.example.com/v2/crawl/id;p=1?skip=1"))+ url = client._client.get.await_args.args[0]+ assert url == "https://api.firecrawl.dev/v2/crawl/id;p=1?skip=1"+++def test_rewrite_refuses_non_absolute_api_url():+ client = AsyncHttpClient(api_key="fc-test-key", api_url="api.firecrawl.dev")+ with pytest.raises(ValueError):+ client._build_url("https://evil.example.com/v2/crawl/id")diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex b20a32096..a6895af43 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -24,6 +24,8 @@ import websocketsimport aiohttpimport asyncio+from ..v2.utils.api_origin import pin_to_api_origin+logger : logging.Logger = logging.getLogger("firecrawl")def get_version():@@ -249,6 +251,7 @@ class V1BatchScrapeStatusResponse(pydantic.BaseModel):expiresAt: datetimenext: Optional[str] = Nonedata: List[V1FirecrawlDocument]+ error: Optional[str] = Noneclass V1CrawlParams(pydantic.BaseModel):"""Parameters for crawling operations."""@@ -1170,7 +1173,7 @@ class V1FirecrawlApp:logger.warning("Expected 'next' URL is missing.")breaktry:- status_response = self._get_request(next_url, headers)+ status_response = self._get_request(pin_to_api_origin(self.api_url, next_url), headers)if status_response.status_code != 200:logger.error(f"Failed to fetch next page: {status_response.status_code}")break@@ -1893,7 +1896,7 @@ class V1FirecrawlApp:logger.warning("Expected 'next' URL is missing.")breaktry:- status_response = self._get_request(next_url, headers)+ status_response = self._get_request(pin_to_api_origin(self.api_url, next_url), headers)if status_response.status_code != 200:logger.error(f"Failed to fetch next page: {status_response.status_code}")break@@ -2538,7 +2541,7 @@ class V1FirecrawlApp:while 'next' in status_data:if len(status_data['data']) == 0:break- status_response = self._get_request(status_data['next'], headers)+ status_response = self._get_request(pin_to_api_origin(self.api_url, status_data['next']), headers)try:status_data = status_response.json()except:@@ -4304,7 +4307,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):if not next_url:logger.warning("Expected 'next' URL is missing.")break- next_data = await self._async_get_request(next_url, headers)+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)data.extend(next_data.get('data', []))status_data = next_datastatus_data['data'] = data@@ -4358,7 +4361,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):if not next_url:logger.warning("Expected 'next' URL is missing.")break- next_data = await self._async_get_request(next_url, headers)+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)data.extend(next_data.get('data', []))status_data = next_datastatus_data['data'] = data@@ -4592,31 +4595,23 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):if not next_url:logger.warning("Expected 'next' URL is missing.")break- next_data = await self._async_get_request(next_url, headers)+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)data.extend(next_data.get('data', []))status_data = next_datastatus_data['data'] = data- response = V1BatchScrapeStatusResponse(+ return V1BatchScrapeStatusResponse(+ success=False if 'error' in status_data else True,status=status_data.get('status'),total=status_data.get('total'),completed=status_data.get('completed'),creditsUsed=status_data.get('creditsUsed'),expiresAt=status_data.get('expiresAt'),- data=status_data.get('data')+ data=status_data.get('data'),+ next=status_data.get('next'),+ error=status_data.get('error'),)- if 'error' in status_data:- response['error'] = status_data['error']-- if 'next' in status_data:- response['next'] = status_data['next']-- return {- 'success': False if 'error' in status_data else True,- **response- }-async def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:"""Get information about errors from an asynchronous batch scrape job.diff --git a/apps/python-sdk/firecrawl/v2/utils/api_origin.py b/apps/python-sdk/firecrawl/v2/utils/api_origin.pynew file mode 100644index 000000000..2c5298ee4--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/utils/api_origin.py@@ -0,0 +1,15 @@+from urllib.parse import urlparse, urlunparse+++def pin_to_api_origin(api_url: str, url: str) -> str:+ """Rewrite an absolute or protocol-relative URL onto api_url's scheme and host so credentials never leave it.++ Relative URLs are returned unchanged. Raises ValueError if api_url is not an absolute URL.+ """+ parsed = urlparse(url)+ if not parsed.netloc:+ return url+ base = urlparse(api_url)+ if not base.scheme or not base.netloc:+ raise ValueError(f"api_url must be an absolute URL, got {api_url!r}")+ return urlunparse((base.scheme, base.netloc, parsed.path or "/", parsed.params, parsed.query, ""))diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.pyindex 7a75ddbb3..4a780cc19 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py@@ -4,8 +4,9 @@ HTTP client utilities for v2 API.import timefrom typing import Dict, Any, Optional-from urllib.parse import urlparse, urlunparse, urljoin+from urllib.parse import urlparse, urljoinimport requests+from .api_origin import pin_to_api_originfrom .get_version import get_versionversion = get_version()@@ -28,27 +29,11 @@ class HttpClient:self.backoff_factor = backoff_factordef _build_url(self, endpoint: str) -> str:- base = urlparse(self.api_url)- ep = urlparse(endpoint)-- # Absolute or protocol-relative (has netloc)- if ep.netloc:- # Different host: keep path/query but force base host/scheme (no token leakage)- path = ep.path or "/"- if (ep.hostname or "") != (base.hostname or ""):- return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))- # Same host: normalize scheme to base- return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))-- # Relative (including leading slash or not)+ if urlparse(endpoint).netloc:+ return pin_to_api_origin(self.api_url, endpoint)base_str = self.api_url if self.api_url.endswith("/") else f"{self.api_url}/"- # Guard protocol-relative like //host/path slipping through as “relative”- if endpoint.startswith("//"):- ep2 = urlparse(f"https:{endpoint}")- path = ep2.path or "/"- return urlunparse((base.scheme or "https", base.netloc, path, "", ep2.query, ""))return urljoin(base_str, endpoint)-+def _prepare_headers(self,idempotency_key: Optional[str] = None,diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.pyindex 52d14623f..b1d270b5e 100644--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py@@ -1,6 +1,7 @@import asyncioimport httpxfrom typing import Optional, Dict, Any+from .api_origin import pin_to_api_originfrom .get_version import get_versionversion = get_version()@@ -35,6 +36,9 @@ class AsyncHttpClient:async def close(self) -> None:await self._client.aclose()+ def _build_url(self, endpoint: str) -> str:+ return pin_to_api_origin(self.api_url, endpoint)+def _headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:headers: Dict[str, str] = {}if idempotency_key:@@ -65,8 +69,9 @@ class AsyncHttpClient:for attempt in range(num_attempts):try:+ url = self._build_url(endpoint)response = await self._client.post(- endpoint,+ url,json=payload,headers={**self._headers(), **(headers or {})},timeout=timeout,@@ -106,8 +111,9 @@ class AsyncHttpClient:for attempt in range(num_attempts):try:+ url = self._build_url(endpoint)response = await self._client.post(- endpoint,+ url,data=data,files=files,headers={**self._headers(), **(headers or {})},@@ -146,8 +152,9 @@ class AsyncHttpClient:for attempt in range(num_attempts):try:+ url = self._build_url(endpoint)response = await self._client.get(- endpoint,+ url,headers={**self._headers(), **(headers or {})},timeout=timeout,)@@ -184,8 +191,9 @@ class AsyncHttpClient:for attempt in range(num_attempts):try:+ url = self._build_url(endpoint)response = await self._client.delete(- endpoint,+ url,headers={**self._headers(), **(headers or {})},timeout=timeout,)@@ -226,8 +234,9 @@ class AsyncHttpClient:for attempt in range(num_attempts):try:+ url = self._build_url(endpoint)response = await self._client.patch(- endpoint,+ url,json=payload,headers={**self._headers(), **(headers or {})},timeout=timeout,