Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): validate PDF download redirects against SSRF (#2127)
Soham Kukreti4138ada0bbb6
diff --git a/crawl4ai/processors/pdf/__init__.py b/crawl4ai/processors/pdf/__init__.pyindex ba61e946..7722fe7f 100644--- a/crawl4ai/processors/pdf/__init__.py+++ b/crawl4ai/processors/pdf/__init__.py@@ -7,6 +7,8 @@ from crawl4ai.models import AsyncCrawlResponse, ScrapingResultfrom crawl4ai.content_scraping_strategy import ContentScrapingStrategyfrom .processor import NaivePDFProcessorStrategy # Assuming your current PDF code is in pdf_processor.py+MAX_PDF_DOWNLOAD_REDIRECTS = 10 # Max redirect hops to follow when downloading a PDF+class PDFCrawlerStrategy(AsyncCrawlerStrategy):"""Crawler strategy for PDF documents.@@ -68,8 +70,10 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):extract_images : bool = False,image_save_dir : str = None,batch_size: int = 4,- logger: AsyncLogger = None):+ logger: AsyncLogger = None,+ url_validator=None):self.logger = logger+ self.url_validator = url_validator # vets the download URL before fetchself.pdf_processor = NaivePDFProcessorStrategy(save_images_locally=save_images_locally,extract_images=extract_images,@@ -164,7 +168,25 @@ class PDFContentScrapingStrategy(ContentScrapingStrategy):# Download PDF with streaming and timeout# Connection timeout: 10s, Read timeout: 300s (5 minutes for large PDFs)- response = requests.get(url, stream=True, timeout=(20, 60 * 10))+ # Redirects are followed manually so url_validator (when set) can vet every hop BEFORE it is fetched+ from urllib.parse import urljoin+ current_url = url+ for _ in range(MAX_PDF_DOWNLOAD_REDIRECTS):+ if self.url_validator:+ self.url_validator(current_url)+ response = requests.get(current_url, stream=True, timeout=(20, 60 * 10),+ allow_redirects=False)+ if response.is_redirect:+ location = response.headers.get("location")+ response.close() # hop response holds its connection open (stream=True)+ if not location:+ raise RuntimeError(f"Redirect without Location header from {current_url}")+ current_url = urljoin(current_url, location)+ continue+ break+ else:+ raise RuntimeError(+ f"Too many redirects (>{MAX_PDF_DOWNLOAD_REDIRECTS}) downloading PDF from {url}")response.raise_for_status()# Get file size if availablediff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex 72928ea1..404d05f1 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -701,6 +701,8 @@ async def handle_crawl_request(if hooks_config:# PDFCrawlerStrategy has no browser page, so hooks can't attach to itraise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")+ # SSRF protection: vet the PDF download URL and every redirect hop+ crawler_config.scraping_strategy.url_validator = validate_url_destination# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inlinecrawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())await crawler.start()@@ -923,6 +925,8 @@ async def handle_stream_crawl_request(if hooks_config:# PDFCrawlerStrategy has no browser page, so hooks can't attach to itraise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")+ # SSRF protection: vet the PDF download URL and every redirect hop+ crawler_config.scraping_strategy.url_validator = validate_url_destination# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inlinecrawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())await crawler.start()diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.pyindex cdce7f53..345c6c42 100644--- a/tests/test_docker_pdf_crawler_pairing.py+++ b/tests/test_docker_pdf_crawler_pairing.py@@ -13,7 +13,7 @@ from unittest.mock import AsyncMock, MagicMockimport pytest-from crawl4ai.processors.pdf import PDFCrawlerStrategy+from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategyROOT = Path(__file__).resolve().parent.parent@@ -86,6 +86,10 @@ async def test_pdf_scraping_strategy_gets_pdf_crawler(api, pool_mock, monkeypatcpool_mock.assert_not_awaited()# The dedicated PDF crawler is not pooled, so the handler must close it.used["crawler"].close.assert_awaited_once()+ # SSRF protection: the handler must wire the server's URL validator into+ # the scraping strategy so every download/redirect hop is vetted.+ effective_config = used["crawler"].arun.await_args.kwargs["config"]+ assert effective_config.scraping_strategy.url_validator is api.validate_url_destination@pytest.mark.asyncio@@ -121,3 +125,82 @@ async def test_default_strategy_still_uses_pool(api, pool_mock):assert response["success"] is Truepool_mock.assert_awaited_once()+++# ---------------------------------------------------------------------------+# PDF download redirect handling (url_validator SSRF guard)+# ---------------------------------------------------------------------------+[email protected](scope="module")+def redirect_server():+ """Real HTTP server: /hop redirects to /doc.pdf, which serves a tiny PDF."""+ import http.server+ import socket+ import threading++ pdf_bytes = (b"%PDF-1.4\n1 0 obj\n<< /Type /Catalog /Pages 2 0 R >>\nendobj\n"+ b"2 0 obj\n<< /Type /Pages /Kids [] /Count 0 >>\nendobj\n"+ b"xref\n0 3\n0000000000 65535 f \n0000000009 00000 n \n"+ b"0000000058 00000 n \ntrailer\n<< /Size 3 /Root 1 0 R >>\n"+ b"startxref\n110\n%%EOF\n")++ class Handler(http.server.BaseHTTPRequestHandler):+ def do_GET(self):+ if self.path == "/hop":+ self.send_response(302)+ self.send_header("Location", "/doc.pdf")+ self.end_headers()+ elif self.path == "/loop":+ self.send_response(302)+ self.send_header("Location", "/loop")+ self.end_headers()+ else:+ self.send_response(200)+ self.send_header("Content-Type", "application/pdf")+ self.end_headers()+ self.wfile.write(pdf_bytes)++ def log_message(self, *args):+ pass++ with socket.socket() as s:+ s.bind(("localhost", 0))+ port = s.getsockname()[1]+ httpd = http.server.ThreadingHTTPServer(("localhost", port), Handler)+ threading.Thread(target=httpd.serve_forever, daemon=True).start()+ yield f"http://localhost:{port}"+ httpd.shutdown()+++def test_download_validator_vets_every_redirect_hop(redirect_server):+ """The validator must see BOTH the original URL and the redirect target,+ and a raising validator must abort the download before the hop is fetched."""+ seen = []++ def validator(u):+ seen.append(u)+ if u.endswith("/doc.pdf"):+ raise ValueError("blocked hop")++ strategy = PDFContentScrapingStrategy(url_validator=validator)+ with pytest.raises(RuntimeError, match="Failed to download"):+ strategy._get_pdf_path(f"{redirect_server}/hop")++ assert seen == [f"{redirect_server}/hop", f"{redirect_server}/doc.pdf"]+++def test_download_redirects_still_followed_without_validator(redirect_server):+ """Back-compat: with no validator, redirects are followed as before."""+ strategy = PDFContentScrapingStrategy()+ path = strategy._get_pdf_path(f"{redirect_server}/hop")+ try:+ assert Path(path).read_bytes().startswith(b"%PDF")+ finally:+ Path(path).unlink(missing_ok=True)+++def test_download_redirect_loop_aborts(redirect_server):+ """An endless redirect chain must abort after the cap, not hang."""+ strategy = PDFContentScrapingStrategy()+ with pytest.raises(RuntimeError, match="[Tt]oo many redirects"):+ strategy._get_pdf_path(f"{redirect_server}/loop")