Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(python-sdk): stop sending the API key to cross-host next URLs (#4861)

Gergő Móricz70a134c4df8d
diff --git a/.github/workflows/test-python-sdk.yml b/.github/workflows/test-python-sdk.yml
new file mode 100644
index 000000000..bbbfcc454
--- /dev/null
+++ b/.github/workflows/test-python-sdk.yml
@@ -0,0 +1,39 @@
+name: Python SDK Test Suite
+
+on:
+ pull_request:
+ branches:
+ - main
+ paths:
+ - apps/python-sdk/**
+ - .github/workflows/test-python-sdk.yml
+
+permissions:
+ contents: read
+
+jobs:
+ unit:
+ name: Unit tests
+ runs-on: blacksmith-2vcpu-ubuntu-2404
+ steps:
+ - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5.1.0
+ with:
+ persist-credentials: false
+
+ - name: Set up Python
+ uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
+ with:
+ python-version: "3.12"
+ cache: pip
+ cache-dependency-path: ./apps/python-sdk/requirements.txt
+
+ - name: Install dependencies
+ working-directory: ./apps/python-sdk
+ run: |
+ python -m pip install --upgrade pip
+ pip install -r requirements.txt
+ pip install -e .
+
+ - name: Run unit tests
+ working-directory: ./apps/python-sdk
+ run: python -m pytest firecrawl/__tests__/unit -q
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 459c68192..6ebce505e 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -27,7 +27,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.45.0"
+__version__ = "4.45.1"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.py b/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.py
new file mode 100644
index 000000000..a32698cfa
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/test_v1_next_url_pinning.py
@@ -0,0 +1,77 @@
+"""The v1 clients must never follow a `next` URL off the configured api_url origin."""
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+from firecrawl.v1.client import AsyncV1FirecrawlApp, V1FirecrawlApp
+
+API_URL = "https://api.firecrawl.dev"
+
+NEXT_URLS = [
+ ("https://evil.example.com/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),
+ ("//evil.example.com/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),
+ ("http://api.firecrawl.dev:8443/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),
+ ("https://evil.example.com/v1/x/id;p=1?skip=1", f"{API_URL}/v1/x/id;p=1?skip=1"),
+ (f"{API_URL}/v1/x/id?skip=1", f"{API_URL}/v1/x/id?skip=1"),
+]
+
+SYNC_CALLS = [
+ ("check_crawl_status", lambda app: app.check_crawl_status("id"), "/v1/crawl/id"),
+ ("check_batch_scrape_status", lambda app: app.check_batch_scrape_status("id"), "/v1/batch/scrape/id"),
+ ("_monitor_job_status", lambda app: app._monitor_job_status("id", {}, 2), "/v1/crawl/id"),
+]
+
+ASYNC_CALLS = [
+ ("check_crawl_status", lambda app: app.check_crawl_status("id"), "/v1/crawl/id"),
+ ("check_batch_scrape_status", lambda app: app.check_batch_scrape_status("id"), "/v1/batch/scrape/id"),
+ ("_async_monitor_job_status", lambda app: app._async_monitor_job_status("id", {}), "/v1/crawl/id"),
+]
+
+
+def _page(markdown, next_url=None):
+ page = {"success": True, "status": "completed", "completed": 2, "total": 2,
+ "creditsUsed": 2, "expiresAt": "2026-10-01T00:00:00Z",
+ "data": [{"markdown": markdown}]}
+ if next_url:
+ page["next"] = next_url
+ return page
+
+
[email protected]("name, call, first_path", SYNC_CALLS, ids=[c[0] for c in SYNC_CALLS])
[email protected]("next_url, expected", NEXT_URLS)
+def test_sync_pagination_pins_next_url(name, call, first_path, next_url, expected):
+ app = V1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)
+ responses = [
+ MagicMock(status_code=200, json=MagicMock(return_value=_page("a", next_url))),
+ MagicMock(status_code=200, json=MagicMock(return_value=_page("b"))),
+ ]
+ with patch("firecrawl.v1.client.requests.get", side_effect=responses) as get:
+ call(app)
+
+ assert [c.args[0] for c in get.call_args_list] == [f"{API_URL}{first_path}", expected]
+
+
[email protected]("name, call, first_path", ASYNC_CALLS, ids=[c[0] for c in ASYNC_CALLS])
[email protected]("next_url, expected", NEXT_URLS)
+def test_async_pagination_pins_next_url(name, call, first_path, next_url, expected):
+ app = AsyncV1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)
+ app._async_get_request = AsyncMock(side_effect=[_page("a", next_url), _page("b")])
+
+ asyncio.run(call(app))
+
+ urls = [c.args[0] for c in app._async_get_request.await_args_list]
+ assert urls == [f"{API_URL}{first_path}", expected]
+
+
+def test_sync_crawl_status_keeps_all_pages():
+ app = V1FirecrawlApp(api_key="fc-test-key", api_url=API_URL)
+ responses = [
+ MagicMock(status_code=200, json=MagicMock(return_value=_page("a", "https://evil.example.com/v1/crawl/id?skip=1"))),
+ MagicMock(status_code=200, json=MagicMock(return_value=_page("b"))),
+ ]
+ with patch("firecrawl.v1.client.requests.get", side_effect=responses):
+ status = app.check_crawl_status("id")
+
+ assert [d.markdown for d in status.data] == ["a", "b"]
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.py
new file mode 100644
index 000000000..453b80d8d
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/utils/test_http_client_async_url.py
@@ -0,0 +1,125 @@
+"""The async client must force cross-host absolute URLs back onto api_url.
+
+Mirrors the sync HttpClient._build_url behavior: a `next` URL returned by the
+API (or a protocol-relative endpoint) must never be followed to another host,
+where the Authorization header would leak the API key.
+"""
+
+import asyncio
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from firecrawl.v2.utils.http_client_async import AsyncHttpClient
+
+
+def _client() -> AsyncHttpClient:
+ client = AsyncHttpClient(
+ api_key="fc-test-key", api_url="https://api.firecrawl.dev"
+ )
+ client._client = MagicMock()
+ response = MagicMock(status_code=200)
+ client._client.get = AsyncMock(return_value=response)
+ client._client.post = AsyncMock(return_value=response)
+ return client
+
+
+def test_get_rewrites_cross_host_next_url():
+ client = _client()
+ asyncio.run(
+ client.get("https://evil.example.com/v2/team/crawl/id?cursor=abc")
+ )
+ url = client._client.get.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id?cursor=abc"
+
+
+def test_get_rewrites_protocol_relative_cross_host_url():
+ client = _client()
+ asyncio.run(client.get("//evil.example.com/v2/team/crawl/id"))
+ url = client._client.get.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"
+
+
+def test_get_keeps_relative_endpoint_untouched():
+ client = _client()
+ asyncio.run(client.get("/v2/team/crawl/id"))
+ url = client._client.get.await_args.args[0]
+ assert url == "/v2/team/crawl/id"
+
+
+def test_get_rewrites_same_host_alternate_port():
+ client = _client()
+ asyncio.run(client.get("https://api.firecrawl.dev:8443/v2/team/crawl/id"))
+ url = client._client.get.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"
+
+
+def test_get_rewrites_same_host_http_url():
+ client = _client()
+ asyncio.run(client.get("http://api.firecrawl.dev/v2/team/crawl/id"))
+ url = client._client.get.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id"
+
+
+def test_post_rewrites_cross_host_url():
+ client = _client()
+ asyncio.run(
+ client.post(
+ "https://evil.example.com/v2/scrape", data={"url": "https://x.test"}
+ )
+ )
+ url = client._client.post.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/scrape"
+
+
+ "verb, call",
+ [
+ ("get", lambda c, url: c.get(url)),
+ ("post", lambda c, url: c.post(url, data={})),
+ ("post", lambda c, url: c.post_multipart(url, data={}, files={})),
+ ("delete", lambda c, url: c.delete(url)),
+ ("patch", lambda c, url: c.patch(url, data={})),
+ ],
+)
+def test_every_verb_rewrites_cross_host_url(verb, call):
+ client = _client()
+ setattr(client._client, verb, AsyncMock(return_value=MagicMock(status_code=200)))
+ asyncio.run(call(client, "https://evil.example.com/v2/team/crawl/id?x=1"))
+ url = getattr(client._client, verb).await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/team/crawl/id?x=1"
+
+
+def test_auto_paginated_crawl_status_never_leaves_api_host():
+ from firecrawl.v2.methods.aio.crawl import get_crawl_status
+
+ client = _client()
+ pages = [
+ {"success": True, "status": "completed", "completed": 2, "total": 2,
+ "creditsUsed": 2, "data": [{"markdown": "a"}],
+ "next": "https://evil.example.com/v2/crawl/id?skip=1"},
+ {"success": True, "status": "completed", "completed": 2, "total": 2,
+ "creditsUsed": 2, "data": [{"markdown": "b"}]},
+ ]
+ client._client.get = AsyncMock(
+ side_effect=[MagicMock(status_code=200, json=MagicMock(return_value=p)) for p in pages]
+ )
+
+ job = asyncio.run(get_crawl_status(client, "id"))
+
+ urls = [c.args[0] for c in client._client.get.await_args_list]
+ assert urls == ["/v2/crawl/id", "https://api.firecrawl.dev/v2/crawl/id?skip=1"]
+ assert [d.markdown for d in job.data] == ["a", "b"]
+
+
+def test_rewrite_keeps_path_params():
+ client = _client()
+ asyncio.run(client.get("https://evil.example.com/v2/crawl/id;p=1?skip=1"))
+ url = client._client.get.await_args.args[0]
+ assert url == "https://api.firecrawl.dev/v2/crawl/id;p=1?skip=1"
+
+
+def test_rewrite_refuses_non_absolute_api_url():
+ client = AsyncHttpClient(api_key="fc-test-key", api_url="api.firecrawl.dev")
+ with pytest.raises(ValueError):
+ client._build_url("https://evil.example.com/v2/crawl/id")
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index b20a32096..a6895af43 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -24,6 +24,8 @@ import websockets
import aiohttp
import asyncio
+from ..v2.utils.api_origin import pin_to_api_origin
+
logger : logging.Logger = logging.getLogger("firecrawl")
def get_version():
@@ -249,6 +251,7 @@ class V1BatchScrapeStatusResponse(pydantic.BaseModel):
expiresAt: datetime
next: Optional[str] = None
data: List[V1FirecrawlDocument]
+ error: Optional[str] = None
class V1CrawlParams(pydantic.BaseModel):
"""Parameters for crawling operations."""
@@ -1170,7 +1173,7 @@ class V1FirecrawlApp:
logger.warning("Expected 'next' URL is missing.")
break
try:
- status_response = self._get_request(next_url, headers)
+ status_response = self._get_request(pin_to_api_origin(self.api_url, next_url), headers)
if status_response.status_code != 200:
logger.error(f"Failed to fetch next page: {status_response.status_code}")
break
@@ -1893,7 +1896,7 @@ class V1FirecrawlApp:
logger.warning("Expected 'next' URL is missing.")
break
try:
- status_response = self._get_request(next_url, headers)
+ status_response = self._get_request(pin_to_api_origin(self.api_url, next_url), headers)
if status_response.status_code != 200:
logger.error(f"Failed to fetch next page: {status_response.status_code}")
break
@@ -2538,7 +2541,7 @@ class V1FirecrawlApp:
while 'next' in status_data:
if len(status_data['data']) == 0:
break
- status_response = self._get_request(status_data['next'], headers)
+ status_response = self._get_request(pin_to_api_origin(self.api_url, status_data['next']), headers)
try:
status_data = status_response.json()
except:
@@ -4304,7 +4307,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
if not next_url:
logger.warning("Expected 'next' URL is missing.")
break
- next_data = await self._async_get_request(next_url, headers)
+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)
data.extend(next_data.get('data', []))
status_data = next_data
status_data['data'] = data
@@ -4358,7 +4361,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
if not next_url:
logger.warning("Expected 'next' URL is missing.")
break
- next_data = await self._async_get_request(next_url, headers)
+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)
data.extend(next_data.get('data', []))
status_data = next_data
status_data['data'] = data
@@ -4592,31 +4595,23 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
if not next_url:
logger.warning("Expected 'next' URL is missing.")
break
- next_data = await self._async_get_request(next_url, headers)
+ next_data = await self._async_get_request(pin_to_api_origin(self.api_url, next_url), headers)
data.extend(next_data.get('data', []))
status_data = next_data
status_data['data'] = data
- response = V1BatchScrapeStatusResponse(
+ return V1BatchScrapeStatusResponse(
+ success=False if 'error' in status_data else True,
status=status_data.get('status'),
total=status_data.get('total'),
completed=status_data.get('completed'),
creditsUsed=status_data.get('creditsUsed'),
expiresAt=status_data.get('expiresAt'),
- data=status_data.get('data')
+ data=status_data.get('data'),
+ next=status_data.get('next'),
+ error=status_data.get('error'),
)
- if 'error' in status_data:
- response['error'] = status_data['error']
-
- if 'next' in status_data:
- response['next'] = status_data['next']
-
- return {
- 'success': False if 'error' in status_data else True,
- **response
- }
-
async def check_batch_scrape_errors(self, id: str) -> V1CrawlErrorsResponse:
"""
Get information about errors from an asynchronous batch scrape job.
diff --git a/apps/python-sdk/firecrawl/v2/utils/api_origin.py b/apps/python-sdk/firecrawl/v2/utils/api_origin.py
new file mode 100644
index 000000000..2c5298ee4
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/utils/api_origin.py
@@ -0,0 +1,15 @@
+from urllib.parse import urlparse, urlunparse
+
+
+def pin_to_api_origin(api_url: str, url: str) -> str:
+ """Rewrite an absolute or protocol-relative URL onto api_url's scheme and host so credentials never leave it.
+
+ Relative URLs are returned unchanged. Raises ValueError if api_url is not an absolute URL.
+ """
+ parsed = urlparse(url)
+ if not parsed.netloc:
+ return url
+ base = urlparse(api_url)
+ if not base.scheme or not base.netloc:
+ raise ValueError(f"api_url must be an absolute URL, got {api_url!r}")
+ return urlunparse((base.scheme, base.netloc, parsed.path or "/", parsed.params, parsed.query, ""))
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client.py b/apps/python-sdk/firecrawl/v2/utils/http_client.py
index 7a75ddbb3..4a780cc19 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client.py
@@ -4,8 +4,9 @@ HTTP client utilities for v2 API.
import time
from typing import Dict, Any, Optional
-from urllib.parse import urlparse, urlunparse, urljoin
+from urllib.parse import urlparse, urljoin
import requests
+from .api_origin import pin_to_api_origin
from .get_version import get_version
version = get_version()
@@ -28,27 +29,11 @@ class HttpClient:
self.backoff_factor = backoff_factor
def _build_url(self, endpoint: str) -> str:
- base = urlparse(self.api_url)
- ep = urlparse(endpoint)
-
- # Absolute or protocol-relative (has netloc)
- if ep.netloc:
- # Different host: keep path/query but force base host/scheme (no token leakage)
- path = ep.path or "/"
- if (ep.hostname or "") != (base.hostname or ""):
- return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
- # Same host: normalize scheme to base
- return urlunparse((base.scheme or "https", base.netloc, path, "", ep.query, ""))
-
- # Relative (including leading slash or not)
+ if urlparse(endpoint).netloc:
+ return pin_to_api_origin(self.api_url, endpoint)
base_str = self.api_url if self.api_url.endswith("/") else f"{self.api_url}/"
- # Guard protocol-relative like //host/path slipping through as “relative”
- if endpoint.startswith("//"):
- ep2 = urlparse(f"https:{endpoint}")
- path = ep2.path or "/"
- return urlunparse((base.scheme or "https", base.netloc, path, "", ep2.query, ""))
return urljoin(base_str, endpoint)
-
+
def _prepare_headers(
self,
idempotency_key: Optional[str] = None,
diff --git a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
index 52d14623f..b1d270b5e 100644
--- a/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
+++ b/apps/python-sdk/firecrawl/v2/utils/http_client_async.py
@@ -1,6 +1,7 @@
import asyncio
import httpx
from typing import Optional, Dict, Any
+from .api_origin import pin_to_api_origin
from .get_version import get_version
version = get_version()
@@ -35,6 +36,9 @@ class AsyncHttpClient:
async def close(self) -> None:
await self._client.aclose()
+ def _build_url(self, endpoint: str) -> str:
+ return pin_to_api_origin(self.api_url, endpoint)
+
def _headers(self, idempotency_key: Optional[str] = None) -> Dict[str, str]:
headers: Dict[str, str] = {}
if idempotency_key:
@@ -65,8 +69,9 @@ class AsyncHttpClient:
for attempt in range(num_attempts):
try:
+ url = self._build_url(endpoint)
response = await self._client.post(
- endpoint,
+ url,
json=payload,
headers={**self._headers(), **(headers or {})},
timeout=timeout,
@@ -106,8 +111,9 @@ class AsyncHttpClient:
for attempt in range(num_attempts):
try:
+ url = self._build_url(endpoint)
response = await self._client.post(
- endpoint,
+ url,
data=data,
files=files,
headers={**self._headers(), **(headers or {})},
@@ -146,8 +152,9 @@ class AsyncHttpClient:
for attempt in range(num_attempts):
try:
+ url = self._build_url(endpoint)
response = await self._client.get(
- endpoint,
+ url,
headers={**self._headers(), **(headers or {})},
timeout=timeout,
)
@@ -184,8 +191,9 @@ class AsyncHttpClient:
for attempt in range(num_attempts):
try:
+ url = self._build_url(endpoint)
response = await self._client.delete(
- endpoint,
+ url,
headers={**self._headers(), **(headers or {})},
timeout=timeout,
)
@@ -226,8 +234,9 @@ class AsyncHttpClient:
for attempt in range(num_attempts):
try:
+ url = self._build_url(endpoint)
response = await self._client.patch(
- endpoint,
+ url,
json=payload,
headers={**self._headers(), **(headers or {})},
timeout=timeout,