Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): let a per-URL config list decide the PDF crawler

Talel Boussetta5ce402f9df60
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index ab1f1aac..9259efee 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -674,6 +674,26 @@ def _load_crawler_configs(
return config_list
+def _needs_pdf_crawler(
+ crawler_config: CrawlerRunConfig,
+ config_list: Optional[List[CrawlerRunConfig]] = None,
+) -> bool:
+ """Whether the crawl runs on PDFCrawlerStrategy. A per-URL list decides when
+ present, since it is what arun_many runs. One crawler serves every URL of a
+ request, so a list mixing PDF and browser entries is refused rather than run
+ on the wrong one."""
+ from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+ configs = config_list or [crawler_config]
+ pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs]
+ if any(pdf) and not all(pdf):
+ raise HTTPException(
+ status_code=400,
+ detail="crawler_configs cannot mix PDFContentScrapingStrategy with other scraping strategies in one request",
+ )
+ return all(pdf)
+
+
def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:
"""Prefix bare hosts with https:// and SSRF-validate every seed URL's
destination. Shared by the streaming and non-streaming crawl handlers so a
@@ -738,33 +758,36 @@ async def handle_crawl_request(
) if config["crawler"]["rate_limiter"]["enabled"] else None
)
+ base_config = config["crawler"]["base_config"]
+ # Per-URL config list: deserialize each and apply base_config. Loaded
+ # before a crawler is chosen, because the list decides which one runs.
+ config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None
+
from crawler_pool import get_crawler, release_crawler
from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
- is_pdf_crawl = isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy)
+ is_pdf_crawl = _needs_pdf_crawler(crawler_config, config_list)
if is_pdf_crawl:
if hooks_config:
# PDFCrawlerStrategy has no browser page, so hooks can't attach to it
raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
# SSRF protection: vet the PDF download URL and every redirect hop
- crawler_config.scraping_strategy.url_validator = validate_url_destination
+ if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+ crawler_config.scraping_strategy.url_validator = validate_url_destination
# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
await crawler.start()
else:
crawler = await get_crawler(browser_config)
-
+
# Attach declarative hooks if provided
hooks_status = {}
if hooks_config:
hooks_status = _attach_declarative_hooks(crawler, hooks_config)
logger.info(f"Hooks attachment status: {hooks_status['status']}")
-
- base_config = config["crawler"]["base_config"]
# Build the config(s) to pass to arun/arun_many
- if crawler_configs:
- # Per-URL config list: deserialize each and apply base_config
- effective_config = _load_crawler_configs(crawler_configs, base_config)
+ if config_list:
+ effective_config = config_list
else:
# Single config (original behavior)
for key, value in base_config.items():
@@ -972,12 +995,13 @@ async def handle_stream_crawl_request(
from crawler_pool import get_crawler
from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
- if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+ if _needs_pdf_crawler(crawler_config, config_list):
if hooks_config:
# PDFCrawlerStrategy has no browser page, so hooks can't attach to it
raise HTTPException(status_code=400, detail="Hooks are not supported with PDFContentScrapingStrategy")
# SSRF protection: vet the PDF download URL and every redirect hop
- crawler_config.scraping_strategy.url_validator = validate_url_destination
+ if isinstance(crawler_config.scraping_strategy, PDFContentScrapingStrategy):
+ crawler_config.scraping_strategy.url_validator = validate_url_destination
# Use PDFCrawlerStrategy when scraping PDFs, as headless Chromium can't render PDFs inline
crawler = AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())
await crawler.start()
diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.py
index b97093df..65303b3d 100644
--- a/deploy/docker/tests/test_crawler_configs_routing.py
+++ b/deploy/docker/tests/test_crawler_configs_routing.py
@@ -4,10 +4,17 @@ The per-URL config list (#1837) was only applied when a non-streaming request
carried two or more URLs. With one URL the handler called arun(), which takes a
single config, and /crawl/stream never passed the list down, so both requests
ran with crawler_config alone and still answered HTTP 200.
+
+With the list applied, it also has to decide the crawler: a list of PDF entries
+runs on PDFCrawlerStrategy, and one crawler cannot serve a list mixing PDF and
+browser entries, so that is refused instead of failing mid-crawl.
"""
import asyncio
+import pytest
+from fastapi import HTTPException
+
from crawl4ai import CrawlerRunConfig
from crawl4ai.models import CrawlResult
@@ -129,3 +136,78 @@ def test_stream_endpoint_forwards_the_list(server_module, monkeypatch):
pass
assert captured["crawler_configs"] == PER_URL
+
+
+PDF = {"type": "PDFContentScrapingStrategy", "params": {}}
+ALL_PDF = [
+ {
+ "type": "CrawlerRunConfig",
+ "params": {"url_matcher": "*.pdf", "scraping_strategy": PDF},
+ },
+]
+MIXED = ALL_PDF + [{"type": "CrawlerRunConfig", "params": {"url_matcher": "*"}}]
+
+
+class RecordingPdfCrawler(RecordingCrawler):
+ """Stands in for AsyncWebCrawler(crawler_strategy=PDFCrawlerStrategy())."""
+
+ def __init__(self, crawler_strategy=None, **kwargs):
+ super().__init__()
+ self.crawler_strategy = crawler_strategy
+
+ async def start(self):
+ return self
+
+ async def close(self):
+ return None
+
+
+def test_a_list_of_pdf_entries_runs_on_the_pdf_crawler(server_module, monkeypatch):
+ from crawl4ai.processors.pdf import PDFCrawlerStrategy
+
+ api = _install(monkeypatch, RecordingCrawler())
+ made = []
+
+ def pdf_crawler(**kwargs):
+ made.append(RecordingPdfCrawler(**kwargs))
+ return made[-1]
+
+ monkeypatch.setattr(api, "AsyncWebCrawler", pdf_crawler)
+
+ asyncio.run(
+ api.handle_crawl_request(
+ urls=["https://example.com/a.pdf"],
+ browser_config={},
+ crawler_config={},
+ config=server_module.config,
+ crawler_configs=ALL_PDF,
+ )
+ )
+
+ [crawler] = made
+ assert isinstance(crawler.crawler_strategy, PDFCrawlerStrategy)
+ [(method, urls, config)] = crawler.calls
+ assert method == "arun_many"
+ # SSRF: every per-URL PDF entry still gets the destination check.
+ assert config[0].scraping_strategy.url_validator is api.validate_url_destination
+
+
[email protected]("stream", [False, True])
+def test_a_list_mixing_pdf_and_browser_entries_is_refused(
+ server_module, monkeypatch, stream
+):
+ api = _install(monkeypatch, RecordingCrawler())
+ handler = api.handle_stream_crawl_request if stream else api.handle_crawl_request
+
+ with pytest.raises(HTTPException) as refused:
+ asyncio.run(
+ handler(
+ urls=["https://example.com/a.pdf", "https://example.com/"],
+ browser_config={},
+ crawler_config={},
+ config=server_module.config,
+ crawler_configs=MIXED,
+ )
+ )
+
+ assert refused.value.status_code == 400
diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py
index 4a888e07..0e5463bb 100644
--- a/tests/test_issue_1837_config_list.py
+++ b/tests/test_issue_1837_config_list.py
@@ -92,7 +92,7 @@ class TestConfigListLogic:
with open("deploy/docker/api.py") as f:
source = f.read()
# Should check crawler_configs and build a list
- assert "if crawler_configs:" in source
+ assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in source
assert "config_list" in source
def test_api_falls_back_to_single_config(self):