Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): close SSRF gap on per-URL configs and dispose PDF crawlers (#2127)

Soham Kukreti2341507d505d
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 404d05f1..09a78393 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -603,12 +603,19 @@ def create_task_response(task: dict, task_id: str, base_url: str) -> dict:
return response
+async def _dispose_crawler(crawler):
+ """Close a dedicated PDF crawler (not pooled) or release a pooled one."""
+ from crawl4ai.processors.pdf import PDFCrawlerStrategy
+ from crawler_pool import release_crawler
+ if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy):
+ await crawler.close()
+ else:
+ await release_crawler(crawler)
+
async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]:
"""Stream results with heartbeats and completion markers."""
import json
from utils import datetime_handler
- from crawler_pool import release_crawler
- from crawl4ai.processors.pdf import PDFCrawlerStrategy
try:
async for result in results_gen:
@@ -636,10 +643,7 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
logger.warning("Client disconnected during streaming")
finally:
if crawler:
- if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy):
- await crawler.close() # not pooled; release_crawler would be a no-op
- else:
- await release_crawler(crawler)
+ await _dispose_crawler(crawler)
def _normalize_and_validate_seeds(urls: List[str]) -> List[str]:
@@ -727,6 +731,9 @@ async def handle_crawl_request(
current_value = getattr(cfg, key)
if current_value is None or current_value == "":
setattr(cfg, key, value)
+ # SSRF: per-URL PDF strategies need the validator wired too
+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
+ cfg.scraping_strategy.url_validator = validate_url_destination
effective_config = config_list
else:
# Single config (original behavior)
@@ -919,7 +926,7 @@ async def handle_stream_crawl_request(
),
)
- from crawler_pool import get_crawler, release_crawler
+ from crawler_pool import get_crawler
from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
if hooks_config:
@@ -964,21 +971,21 @@ async def handle_stream_crawl_request(
except (UntrustedConfigError, HookValidationError) as e:
if crawler:
- await release_crawler(crawler)
+ await _dispose_crawler(crawler)
raise HTTPException(status_code=400, detail=f"Rejected request: {e}")
except HTTPException:
# Deliberate status (e.g. 400 SSRF "URL blocked") must pass through
# rather than be genericized to 500 by the handler below.
if crawler:
- await release_crawler(crawler)
+ await _dispose_crawler(crawler)
raise
except Exception as e:
- # Release crawler on setup error (for successful streams,
- # release happens in stream_results finally block)
+ # Dispose crawler on setup error (for successful streams,
+ # disposal happens in stream_results finally block)
if crawler:
- await release_crawler(crawler)
+ await _dispose_crawler(crawler)
logger.error(f"Stream crawl error: {str(e)}", exc_info=True)
# Raising HTTPException here will prevent streaming response
raise HTTPException(
diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.py
index 1fe18762..952f1daa 100644
--- a/tests/test_issue_2127_docker_pdf.py
+++ b/tests/test_issue_2127_docker_pdf.py
@@ -31,16 +31,16 @@ async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch)
monkeypatch.syspath_prepend(str(docker_dir))
api = importlib.import_module("api")
- crawler_pool = importlib.import_module("crawler_pool")
egress_broker = importlib.import_module("egress_broker")
governor = importlib.import_module("governor")
crawler = MagicMock()
crawler.arun_many = AsyncMock(return_value=MagicMock())
+ crawler.start = AsyncMock()
monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls)
monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None)
monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None)
- monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler))
+ monkeypatch.setattr(api, "AsyncWebCrawler", MagicMock(return_value=crawler))
crawler_config = {
"type": "CrawlerRunConfig",