Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
test(docker): cover the per-URL crawler_configs PDF SSRF guard
ntohidid57a31c8d2e7
diff --git a/tests/test_docker_pdf_crawler_pairing.py b/tests/test_docker_pdf_crawler_pairing.pyindex 345c6c42..20e94ed1 100644--- a/tests/test_docker_pdf_crawler_pairing.py+++ b/tests/test_docker_pdf_crawler_pairing.py@@ -50,6 +50,7 @@ def pool_mock(api, monkeypatch):pooled = MagicMock()pooled.arun = AsyncMock(return_value=[])+ pooled.arun_many = AsyncMock(return_value=[]) # per-URL config list pathpooled.active_requests = 1 # release_crawler decrements this intmock = AsyncMock(return_value=pooled)monkeypatch.setattr(crawler_pool, "get_crawler", mock)@@ -127,6 +128,40 @@ async def test_default_strategy_still_uses_pool(api, pool_mock):pool_mock.assert_awaited_once()+async def test_per_url_pdf_strategy_gets_validator(api, pool_mock):+ """SSRF: a PDF strategy sent per-URL via crawler_configs must be vetted too.++ crawler_configs is a public per-URL field on /crawl, and the handler builds+ that config list on a separate branch from the top-level config. Wiring+ url_validator only on the top-level branch leaves the per-URL one doing an+ unvalidated download of whatever the request names.+ """+ pdf_config = _crawler_config_payload(with_pdf_strategy=True)+ plain_config = _crawler_config_payload(with_pdf_strategy=False)++ response = await api.handle_crawl_request(+ # The config list branch only engages with more than one URL.+ urls=["https://example.com/document.pdf", "https://example.com/page.html"],+ browser_config={"type": "BrowserConfig", "params": {}},+ crawler_config=plain_config, # top-level is clean; the PDF rides per-URL+ crawler_configs=[pdf_config, plain_config],+ config=CONFIG,+ )++ assert response["success"] is True+ configs = pool_mock.return_value.arun_many.await_args.kwargs["config"]+ pdf_strategies = [+ cfg.scraping_strategy for cfg in configs+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy)+ ]+ # Guard the guard: if deserialization ever drops the strategy, the loop+ # below would pass vacuously and the test would protect nothing.+ assert len(pdf_strategies) == 1+ for strategy in pdf_strategies:+ assert strategy.url_validator is api.validate_url_destination++# ---------------------------------------------------------------------------# PDF download redirect handling (url_validator SSRF guard)# ---------------------------------------------------------------------------