Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

test(docker): cover the per-URL crawler_configs PDF SSRF guard

ntohidid57a31c8d2e7
diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.py
index 345c6c42..20e94ed1 100644
--- a/tests/test_docker_pdf_crawler_pairing.py
+++ b/tests/test_docker_pdf_crawler_pairing.py
@@ -50,6 +50,7 @@ def pool_mock(api, monkeypatch):
pooled = MagicMock()
pooled.arun = AsyncMock(return_value=[])
+ pooled.arun_many = AsyncMock(return_value=[]) # per-URL config list path
pooled.active_requests = 1 # release_crawler decrements this int
mock = AsyncMock(return_value=pooled)
monkeypatch.setattr(crawler_pool, "get_crawler", mock)
@@ -127,6 +128,40 @@ async def test_default_strategy_still_uses_pool(api, pool_mock):
pool_mock.assert_awaited_once()
+async def test_per_url_pdf_strategy_gets_validator(api, pool_mock):
+ """SSRF: a PDF strategy sent per-URL via crawler_configs must be vetted too.
+
+ crawler_configs is a public per-URL field on /crawl, and the handler builds
+ that config list on a separate branch from the top-level config. Wiring
+ url_validator only on the top-level branch leaves the per-URL one doing an
+ unvalidated download of whatever the request names.
+ """
+ pdf_config = _crawler_config_payload(with_pdf_strategy=True)
+ plain_config = _crawler_config_payload(with_pdf_strategy=False)
+
+ response = await api.handle_crawl_request(
+ # The config list branch only engages with more than one URL.
+ urls=["https://example.com/document.pdf", "https://example.com/page.html"],
+ browser_config={"type": "BrowserConfig", "params": {}},
+ crawler_config=plain_config, # top-level is clean; the PDF rides per-URL
+ crawler_configs=[pdf_config, plain_config],
+ config=CONFIG,
+ )
+
+ assert response["success"] is True
+ configs = pool_mock.return_value.arun_many.await_args.kwargs["config"]
+ pdf_strategies = [
+ cfg.scraping_strategy for cfg in configs
+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy)
+ ]
+ # Guard the guard: if deserialization ever drops the strategy, the loop
+ # below would pass vacuously and the test would protect nothing.
+ assert len(pdf_strategies) == 1
+ for strategy in pdf_strategies:
+ assert strategy.url_validator is api.validate_url_destination
+
+
# ---------------------------------------------------------------------------
# PDF download redirect handling (url_validator SSRF guard)
# ---------------------------------------------------------------------------