Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): close SSRF gap on per-URL configs and dispose PDF crawlers (#2127)
Soham Kukreti2341507d505d
diff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex 404d05f1..09a78393 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -603,12 +603,19 @@ def create_task_response(task: dict, task_id: str, base_url: str) -> dict:return response+async def _dispose_crawler(crawler):+ """Close a dedicated PDF crawler (not pooled) or release a pooled one."""+ from crawl4ai.processors.pdf import PDFCrawlerStrategy+ from crawler_pool import release_crawler+ if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy):+ await crawler.close()+ else:+ await release_crawler(crawler)+async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator) -> AsyncGenerator[bytes, None]:"""Stream results with heartbeats and completion markers."""import jsonfrom utils import datetime_handler- from crawler_pool import release_crawler- from crawl4ai.processors.pdf import PDFCrawlerStrategytry:async for result in results_gen:@@ -636,10 +643,7 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)logger.warning("Client disconnected during streaming")finally:if crawler:- if isinstance(crawler.crawler_strategy, PDFCrawlerStrategy):- await crawler.close() # not pooled; release_crawler would be a no-op- else:- await release_crawler(crawler)+ await _dispose_crawler(crawler)def _normalize_and_validate_seeds(urls: List[str]) -> List[str]:@@ -727,6 +731,9 @@ async def handle_crawl_request(current_value = getattr(cfg, key)if current_value is None or current_value == "":setattr(cfg, key, value)+ # SSRF: per-URL PDF strategies need the validator wired too+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):+ cfg.scraping_strategy.url_validator = validate_url_destinationeffective_config = config_listelse:# Single config (original behavior)@@ -919,7 +926,7 @@ async def handle_stream_crawl_request(),)- from crawler_pool import get_crawler, release_crawler+ from crawler_pool import get_crawlerfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategyif isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):if hooks_config:@@ -964,21 +971,21 @@ async def handle_stream_crawl_request(except (UntrustedConfigError, HookValidationError) as e:if crawler:- await release_crawler(crawler)+ await _dispose_crawler(crawler)raise HTTPException(status_code=400, detail=f"Rejected request: {e}")except HTTPException:# Deliberate status (e.g. 400 SSRF "URL blocked") must pass through# rather than be genericized to 500 by the handler below.if crawler:- await release_crawler(crawler)+ await _dispose_crawler(crawler)raiseexcept Exception as e:- # Release crawler on setup error (for successful streams,- # release happens in stream_results finally block)+ # Dispose crawler on setup error (for successful streams,+ # disposal happens in stream_results finally block)if crawler:- await release_crawler(crawler)+ await _dispose_crawler(crawler)logger.error(f"Stream crawl error: {str(e)}", exc_info=True)# Raising HTTPException here will prevent streaming responseraise HTTPException(diff --git a/tests/test_issue_2127_docker_pdf.py b/tests/test_issue_2127_docker_pdf.pyindex 1fe18762..952f1daa 100644--- a/tests/test_issue_2127_docker_pdf.py+++ b/tests/test_issue_2127_docker_pdf.py@@ -31,16 +31,16 @@ async def test_stream_handler_preserves_requested_scraping_strategy(monkeypatch)monkeypatch.syspath_prepend(str(docker_dir))api = importlib.import_module("api")- crawler_pool = importlib.import_module("crawler_pool")egress_broker = importlib.import_module("egress_broker")governor = importlib.import_module("governor")crawler = MagicMock()crawler.arun_many = AsyncMock(return_value=MagicMock())+ crawler.start = AsyncMock()monkeypatch.setattr(api, "_normalize_and_validate_seeds", lambda urls: urls)monkeypatch.setattr(egress_broker, "enforce_egress", lambda _: None)monkeypatch.setattr(governor, "clamp_deep_crawl", lambda _: None)- monkeypatch.setattr(crawler_pool, "get_crawler", AsyncMock(return_value=crawler))+ monkeypatch.setattr(api, "AsyncWebCrawler", MagicMock(return_value=crawler))crawler_config = {"type": "CrawlerRunConfig",