Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): validate PDF download redirects against SSRF (#2127)

Soham Kukreti4138ada0bbb6
diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.py
index ba61e946..7722fe7f 100644
--- a/crawl4ai/processors/pdf/__init__.py
+++ b/crawl4ai/processors/pdf/__init__.py
@@ -7,6 +7,8 @@ from crawl4ai.models import AsyncCrawlResponse, ScrapingResult
from crawl4ai.content_scraping_strategy import ContentScrapingStrategy
from .processor import NaivePDFProcessorStrategy # Assuming your current PDF code is in pdf_processor.py
+MAX_PDF_DOWNLOAD_REDIRECTS = 10 # Max redirect hops to follow when downloading a PDF
+
class PDFCrawlerStrategy(AsyncCrawlerStrategy):
"""Crawler strategy for PDF documents.
@@ -68,8 +70,10 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):
extract_images : bool = False,
image_save_dir : str = None,
batch_size: int = 4,
- logger: AsyncLogger = None):
+ logger: AsyncLogger = None,
+ url_validator=None):
self.logger = logger
+ self.url_validator = url_validator # vets the download URL before fetch
self.pdf_processor = NaivePDFProcessorStrategy(
save_images_locally=save_images_locally,
extract_images=extract_images,
@@ -164,7 +168,25 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):
# Download PDF with streaming and timeout
# Connection timeout: 10s, Read timeout: 300s (5 minutes for large PDFs)
- response = requests.get(url, stream=True, timeout=(20, 60 * 10))
+ # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched
+ from urllib.parse import urljoin
+ current_url = url
+ for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS):
+ if self.url_validator:
+ self.url_validator(current_url)
+ response = requests.get(current_url, stream=True, timeout=(20, 60 * 10),
+ allow_redirects=False)
+ if response.is_redirect:
+ location = response.headers.get("location")
+ response.close() # hop response holds its connection open (stream=True)
+ if not location:
+ raise RuntimeError(f"Redirect without Location header from {current_url}")
+ current_url = urljoin(current_url, location)
+ continue
+ break
+ else:
+ raise RuntimeError(
+ f"Too many redirects (>{MAX_PDF_DOWNLOAD_REDIRECTS}) downloading PDF from {url}")
response.raise_for_status()
# Get file size if available
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 72928ea1..404d05f1 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -701,6 +701,8 @@ async def handle_crawl_request(
if hooks_config:
# PDFCrawlerStrategy has no browser page, so hooks can't attach to it
raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
+ # SSRF protection: vet the PDF download URL and every redirect hop
+ crawler_config.scraping_strategy.url_validator = validate_url_destination
# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
await crawler.start()
@@ -923,6 +925,8 @@ async def handle_stream_crawl_request(
if hooks_config:
# PDFCrawlerStrategy has no browser page, so hooks can't attach to it
raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
+ # SSRF protection: vet the PDF download URL and every redirect hop
+ crawler_config.scraping_strategy.url_validator = validate_url_destination
# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
await crawler.start()
diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py
index cdce7f53..345c6c42 100644
--- a/tests/test_docker_pdf_crawler_pairing.py
+++ b/tests/test_docker_pdf_crawler_pairing.py
@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMock
import pytest
-from crawl4ai.processors.pdf import PDFCrawlerStrategy
+from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
ROOT = Path(__file__).resolve().parent.parent
@@ -86,6 +86,10 @@ async def test_pdf_scraping_strategy_gets_pdf_crawler(api, pool_mock, monkeypatc
pool_mock.assert_not_awaited()
# The dedicated PDF crawler is not pooled, so the handler must close it.
used["crawler"].close.assert_awaited_once()
+ # SSRF protection: the handler must wire the server's URL validator into
+ # the scraping strategy so every download/redirect hop is vetted.
+ effective_config = used["crawler"].arun.await_args.kwargs["config"]
+ assert effective_config.scraping_strategy.url_validator is api.validate_url_destination
@pytest.mark.asyncio
@@ -121,3 +125,82 @@ async def test_default_strategy_still_uses_pool(api, pool_mock):
assert response["success"] is True
pool_mock.assert_awaited_once()
+
+
+# ---------------------------------------------------------------------------
+# PDF download redirect handling (url_validator SSRF guard)
+# ---------------------------------------------------------------------------
+
[email protected](scope="module")
+def redirect_server():
+ """Real HTTP server: /hop redirects to /doc.pdf, which serves a tiny PDF."""
+ import http.server
+ import socket
+ import threading
+
+ pdf_bytes = (b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"
+ b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"
+ b"xref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n"
+ b"0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\n"
+ b"startxref\n110\n%%EOF\n")
+
+ class Handler(http.server.BaseHTTPRequestHandler):
+ def do_GET(self):
+ if self.path == "/hop":
+ self.send_response(302)
+ self.send_header("Location", "/doc.pdf")
+ self.end_headers()
+ elif self.path == "/loop":
+ self.send_response(302)
+ self.send_header("Location", "/loop")
+ self.end_headers()
+ else:
+ self.send_response(200)
+ self.send_header("Content-Type", "application/pdf")
+ self.end_headers()
+ self.wfile.write(pdf_bytes)
+
+ def log_message(self, *args):
+ pass
+
+ with socket.socket() as s:
+ s.bind(("localhost", 0))
+ port = s.getsockname()[1]
+ httpd = http.server.ThreadingHTTPServer(("localhost", port), Handler)
+ threading.Thread(target=httpd.serve_forever, daemon=True).start()
+ yield f"http://localhost:{port}"
+ httpd.shutdown()
+
+
+def test_download_validator_vets_every_redirect_hop(redirect_server):
+ """The validator must see BOTH the original URL and the redirect target,
+ and a raising validator must abort the download before the hop is fetched."""
+ seen = []
+
+ def validator(u):
+ seen.append(u)
+ if u.endswith("/doc.pdf"):
+ raise ValueError("blocked hop")
+
+ strategy = PDFContentScrapingStrategy(url_validator=validator)
+ with pytest.raises(RuntimeError, match="Failed to download"):
+ strategy._get_pdf_path(f"{redirect_server}/hop")
+
+ assert seen == [f"{redirect_server}/hop", f"{redirect_server}/doc.pdf"]
+
+
+def test_download_redirects_still_followed_without_validator(redirect_server):
+ """Back-compat: with no validator, redirects are followed as before."""
+ strategy = PDFContentScrapingStrategy()
+ path = strategy._get_pdf_path(f"{redirect_server}/hop")
+ try:
+ assert Path(path).read_bytes().startswith(b"%PDF")
+ finally:
+ Path(path).unlink(missing_ok=True)
+
+
+def test_download_redirect_loop_aborts(redirect_server):
+ """An endless redirect chain must abort after the cap, not hang."""
+ strategy = PDFContentScrapingStrategy()
+ with pytest.raises(RuntimeError, match="[Tt]oo many redirects"):
+ strategy._get_pdf_path(f"{redirect_server}/loop")