Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): let a per-URL config list decide the PDF crawler
Talel Boussetta5ce402f9df60
diff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex ab1f1aac..9259efee 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -674,6 +674,26 @@ def _load_crawler_configs(return config_list+def _needs_pdf_crawler(+ crawler_config: CrawlerRunConfig,+ config_list: Optional[List[CrawlerRunConfig]] = None,+) -> bool:+ """Whether the crawl runs on PDFCrawlerStrategy. A per-URL list decides when+ present, since it is what arun_many runs. One crawler serves every URL of a+ request, so a list mixing PDF and browser entries is refused rather than run+ on the wrong one."""+ from crawl4ai.processors.pdf import PDFContentScrapingStrategy++ configs = config_list or [crawler_config]+ pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs]+ if any(pdf) and not all(pdf):+ raise HTTPException(+ status_code=400,+ detail="crawler_configs cannot mix PDFContentScrapingStrategy with other scraping strategies in one request",+ )+ return all(pdf)++def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:"""Prefix bare hosts with https:// and SSRF-validate every seed URL'sdestination. Shared by the streaming and non-streaming crawl handlers so a@@ -738,33 +758,36 @@ async def handle_crawl_request() if config["crawler"]["rate_limiter"]["enabled"] else None)+ base_config = config["crawler"]["base_config"]+ # Per-URL config list: deserialize each and apply base_config. Loaded+ # before a crawler is chosen, because the list decides which one runs.+ config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None+from crawler_pool import get_crawler, release_crawlerfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy- is_pdf_crawl = isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy)+ is_pdf_crawl = _needs_pdf_crawler(crawler_config, config_list)if is_pdf_crawl:if hooks_config:# PDFCrawlerStrategy has no browser page, so hooks can't attach to itraise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")# SSRF protection: vet the PDF download URL and every redirect hop- crawler_config.scraping_strategy.url_validator = validate_url_destination+ if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):+ crawler_config.scraping_strategy.url_validator = validate_url_destination# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inlinecrawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())await crawler.start()else:crawler = await get_crawler(browser_config)-+# Attach declarative hooks if providedhooks_status = {}if hooks_config:hooks_status = _attach_declarative_hooks(crawler, hooks_config)logger.info(f"Hooks attachment status: {hooks_status['status']}")-- base_config = config["crawler"]["base_config"]# Build the config(s) to pass to arun/arun_many- if crawler_configs:- # Per-URL config list: deserialize each and apply base_config- effective_config = _load_crawler_configs(crawler_configs, base_config)+ if config_list:+ effective_config = config_listelse:# Single config (original behavior)for key, value in base_config.items():@@ -972,12 +995,13 @@ async def handle_stream_crawl_request(from crawler_pool import get_crawlerfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy- if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):+ if _needs_pdf_crawler(crawler_config, config_list):if hooks_config:# PDFCrawlerStrategy has no browser page, so hooks can't attach to itraise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")# SSRF protection: vet the PDF download URL and every redirect hop- crawler_config.scraping_strategy.url_validator = validate_url_destination+ if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):+ crawler_config.scraping_strategy.url_validator = validate_url_destination# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inlinecrawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())await crawler.start()diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.pyindex b97093df..65303b3d 100644--- a/deploy/docker/tests/test_crawler_configs_routing.py+++ b/deploy/docker/tests/test_crawler_configs_routing.py@@ -4,10 +4,17 @@ The per-URL config list (#1837) was only applied when a non-streaming requestcarried two or more URLs. With one URL the handler called arun(), which takes asingle config, and /crawl/stream never passed the list down, so both requestsran with crawler_config alone and still answered HTTP 200.++With the list applied, it also has to decide the crawler: a list of PDF entries+runs on PDFCrawlerStrategy, and one crawler cannot serve a list mixing PDF and+browser entries, so that is refused instead of failing mid-crawl."""import asyncio+import pytest+from fastapi import HTTPException+from crawl4ai import CrawlerRunConfigfrom crawl4ai.models import CrawlResult@@ -129,3 +136,78 @@ def test_stream_endpoint_forwards_the_list(server_module, monkeypatch):passassert captured["crawler_configs"] == PER_URL+++PDF = {"type": "PDFContentScrapingStrategy", "params": {}}+ALL_PDF = [+ {+ "type": "CrawlerRunConfig",+ "params": {"url_matcher": "*.pdf", "scraping_strategy": PDF},+ },+]+MIXED = ALL_PDF + [{"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}}]+++class RecordingPdfCrawler(RecordingCrawler):+ """Stands in for AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())."""++ def __init__(self, crawler_strategy=None, **kwargs):+ super().__init__()+ self.crawler_strategy = crawler_strategy++ async def start(self):+ return self++ async def close(self):+ return None+++def test_a_list_of_pdf_entries_runs_on_the_pdf_crawler(server_module, monkeypatch):+ from crawl4ai.processors.pdf import PDFCrawlerStrategy++ api = _install(monkeypatch, RecordingCrawler())+ made = []++ def pdf_crawler(**kwargs):+ made.append(RecordingPdfCrawler(**kwargs))+ return made[-1]++ monkeypatch.setattr(api, "AsyncWebCrawler", pdf_crawler)++ asyncio.run(+ api.handle_crawl_request(+ urls=["https://example.com/a.pdf"],+ browser_config={},+ crawler_config={},+ config=server_module.config,+ crawler_configs=ALL_PDF,+ )+ )++ [crawler] = made+ assert isinstance(crawler.crawler_strategy, PDFCrawlerStrategy)+ [(method, urls, config)] = crawler.calls+ assert method == "arun_many"+ # SSRF: every per-URL PDF entry still gets the destination check.+ assert config[0].scraping_strategy.url_validator is api.validate_url_destination++[email protected]("stream", [False, True])+def test_a_list_mixing_pdf_and_browser_entries_is_refused(+ server_module, monkeypatch, stream+):+ api = _install(monkeypatch, RecordingCrawler())+ handler = api.handle_stream_crawl_request if stream else api.handle_crawl_request++ with pytest.raises(HTTPException) as refused:+ asyncio.run(+ handler(+ urls=["https://example.com/a.pdf", "https://example.com/"],+ browser_config={},+ crawler_config={},+ config=server_module.config,+ crawler_configs=MIXED,+ )+ )++ assert refused.value.status_code == 400diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.pyindex 4a888e07..0e5463bb 100644--- a/tests/test_issue_1837_config_list.py+++ b/tests/test_issue_1837_config_list.py@@ -92,7 +92,7 @@ class TestConfigListLogic:with open("deploy/docker/api.py") as f:source = f.read()# Should check crawler_configs and build a list- assert "if crawler_configs:" in source+ assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in sourceassert "config_list" in sourcedef test_api_falls_back_to_single_config(self):