Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(docker): apply crawler_configs to single-URL and streaming requests
Talel Boussetta81efcc35f00a
diff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex b03841e7..ab1f1aac 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -648,6 +648,32 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)await _dispose_crawler(crawler)+def _load_crawler_configs(+ crawler_configs: List[dict],+ base_config: Optional[dict] = None,+ stream: bool = False,+) -> List[CrawlerRunConfig]:+ """Deserialize a per-URL crawler_configs list under the untrusted gate.+ Shared by the streaming and non-streaming crawl handlers so both apply the+ same boundary and wire the PDF URL validator into every entry."""+ from crawl4ai.processors.pdf import PDFContentScrapingStrategy++ config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]+ for cfg in config_list:+ for key, value in (base_config or {}).items():+ if hasattr(cfg, key):+ current_value = getattr(cfg, key)+ if current_value is None or current_value == "":+ setattr(cfg, key, value)+ # SSRF: per-URL PDF strategies need the validator wired too+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):+ cfg.scraping_strategy.url_validator = validate_url_destination+ if stream:+ # arun_many takes the stream flag from the first config of a list.+ cfg.stream = True+ return config_list++def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:"""Prefix bare hosts with https:// and SSRF-validate every seed URL'sdestination. Shared by the streaming and non-streaming crawl handlers so a@@ -736,19 +762,9 @@ async def handle_crawl_request(base_config = config["crawler"]["base_config"]# Build the config(s) to pass to arun/arun_many- if crawler_configs and len(urls) > 1:+ if crawler_configs:# Per-URL config list: deserialize each and apply base_config- config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]- for cfg in config_list:- for key, value in base_config.items():- if hasattr(cfg, key):- current_value = getattr(cfg, key)- if current_value is None or current_value == "":- setattr(cfg, key, value)- # SSRF: per-URL PDF strategies need the validator wired too- if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):- cfg.scraping_strategy.url_validator = validate_url_destination- effective_config = config_list+ effective_config = _load_crawler_configs(crawler_configs, base_config)else:# Single config (original behavior)for key, value in base_config.items():@@ -759,9 +775,12 @@ async def handle_crawl_request(effective_config = crawler_configresults = []- func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")+ # arun takes one config, so a per-URL list goes to arun_many even for+ # a single URL; arun_many matches each URL against the list.+ use_many = len(urls) > 1 or isinstance(effective_config, list)+ func = getattr(crawler, "arun_many" if use_many else "arun")partial_func = partial(func,- urls[0] if len(urls) == 1 else urls,+ urls if use_many else urls[0],config=effective_config,dispatcher=dispatcher)# Optional per-crawl wall-clock deadline (config limits.wall_clock_s; 0 = none).@@ -911,7 +930,8 @@ async def handle_stream_crawl_request(browser_config: dict,crawler_config: dict,config: dict,- hooks_config: Optional[dict] = None+ hooks_config: Optional[dict] = None,+ crawler_configs: Optional[List[dict]] = None,) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]:"""Handle streaming crawl requests with optional hooks."""hooks_info = None@@ -936,6 +956,9 @@ async def handle_stream_crawl_request(clamp_deep_crawl(crawler_config)crawler_config.stream = True+ config_list = (+ _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None+ )# Deep crawl streaming supports exactly one start URLif crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1:@@ -984,7 +1007,7 @@ async def handle_stream_crawl_request()results_gen = await crawler.arun_many(urls=urls,- config=crawler_config,+ config=config_list or crawler_config,dispatcher=dispatcher)diff --git a/deploy/docker/server.py b/deploy/docker/server.pyindex 3d71a307..5f08bae4 100644--- a/deploy/docker/server.py+++ b/deploy/docker/server.py@@ -1032,7 +1032,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):browser_config=crawl_request.browser_config,crawler_config=crawl_request.crawler_config,config=config,- hooks_config=hooks_config+ hooks_config=hooks_config,+ crawler_configs=crawl_request.crawler_configs,)# Add hooks info to response headers if availablediff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.pynew file mode 100644index 00000000..b97093df--- /dev/null+++ b/deploy/docker/tests/test_crawler_configs_routing.py@@ -0,0 +1,131 @@+"""crawler_configs must reach arun_many for one URL and on the streaming path.++The per-URL config list (#1837) was only applied when a non-streaming request+carried two or more URLs. With one URL the handler called arun(), which takes a+single config, and /crawl/stream never passed the list down, so both requests+ran with crawler_config alone and still answered HTTP 200.+"""++import asyncio++from crawl4ai import CrawlerRunConfig+from crawl4ai.models import CrawlResult++PER_URL = [+ {+ "type": "CrawlerRunConfig",+ "params": {"url_matcher": "*example.com*", "word_count_threshold": 7},+ },+]+++class RecordingCrawler:+ def __init__(self):+ self.calls = []++ async def arun(self, url, config=None, **kwargs):+ self.calls.append(("arun", url, config))+ return CrawlResult(url=url, html="", success=True)++ async def arun_many(self, urls, config=None, **kwargs):+ self.calls.append(("arun_many", urls, config))+ return [CrawlResult(url=url, html="", success=True) for url in urls]+++def _install(monkeypatch, crawler):+ import api+ import crawler_pool++ async def get_crawler(*args, **kwargs):+ return crawler++ async def release_crawler(*args, **kwargs):+ return None++ # Offline: the seed check resolves DNS.+ monkeypatch.setattr(api, "validate_url_destination", lambda url: None)+ monkeypatch.setattr(crawler_pool, "get_crawler", get_crawler)+ monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)+ return api+++def test_single_url_uses_the_config_list(server_module, monkeypatch):+ crawler = RecordingCrawler()+ api = _install(monkeypatch, crawler)++ asyncio.run(+ api.handle_crawl_request(+ urls=["https://example.com/"],+ browser_config={},+ crawler_config={},+ config=server_module.config,+ crawler_configs=PER_URL,+ )+ )++ [(method, urls, config)] = crawler.calls+ assert method == "arun_many"+ assert urls == ["https://example.com/"]+ assert [c.word_count_threshold for c in config] == [7]+++def test_single_url_without_a_list_still_uses_arun(server_module, monkeypatch):+ crawler = RecordingCrawler()+ api = _install(monkeypatch, crawler)++ asyncio.run(+ api.handle_crawl_request(+ urls=["https://example.com/"],+ browser_config={},+ crawler_config={},+ config=server_module.config,+ )+ )++ [(method, url, config)] = crawler.calls+ assert method == "arun"+ assert url == "https://example.com/"+ assert isinstance(config, CrawlerRunConfig)+++def test_stream_passes_the_config_list(server_module, monkeypatch):+ crawler = RecordingCrawler()+ api = _install(monkeypatch, crawler)++ asyncio.run(+ api.handle_stream_crawl_request(+ urls=["https://example.com/", "https://example.org/"],+ browser_config={},+ crawler_config={},+ config=server_module.config,+ crawler_configs=PER_URL,+ )+ )++ [(method, urls, config)] = crawler.calls+ assert method == "arun_many"+ assert [c.word_count_threshold for c in config] == [7]+ # arun_many takes the stream flag from the first config of a list.+ assert all(c.stream for c in config)+++def test_stream_endpoint_forwards_the_list(server_module, monkeypatch):+ captured = {}++ async def fake_handle_stream(**kwargs):+ captured.update(kwargs)+ raise RuntimeError("stop after capture")++ monkeypatch.setattr(+ server_module, "handle_stream_crawl_request", fake_handle_stream+ )+ request = server_module.CrawlRequestWithHooks(+ urls=["https://example.com/"], crawler_configs=PER_URL+ )++ try:+ asyncio.run(server_module.stream_process(crawl_request=request))+ except RuntimeError:+ pass++ assert captured["crawler_configs"] == PER_URLdiff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.pyindex 45150e58..4a888e07 100644--- a/tests/test_issue_1837_config_list.py+++ b/tests/test_issue_1837_config_list.py@@ -88,11 +88,11 @@ class TestConfigListLogic:"""Verify the branching logic for single vs list configs."""def test_api_uses_config_list_when_provided(self):- """When crawler_configs is provided with multiple URLs, it should be used."""+ """When crawler_configs is provided, it should be used, for one URL or many."""with open("deploy/docker/api.py") as f:source = f.read()# Should check crawler_configs and build a list- assert "if crawler_configs and len(urls) > 1:" in source+ assert "if crawler_configs:" in sourceassert "config_list" in sourcedef test_api_falls_back_to_single_config(self):@@ -125,12 +125,14 @@ class TestServerEndpoint:class TestBackwardCompatibility:"""Ensure existing single-config requests still work."""- def test_single_url_ignores_crawler_configs(self):- """With a single URL, crawler_configs should be ignored (uses arun, not arun_many)."""+ def test_single_url_with_config_list_uses_arun_many(self):+ """A config list goes to arun_many even for a single URL (arun takes one config).++ Behavioral coverage: deploy/docker/tests/test_crawler_configs_routing.py.+ """with open("deploy/docker/api.py") as f:source = f.read()- # Single URL uses arun which only takes one config- assert '"arun" if len(urls) == 1 else "arun_many"' in source+ assert "use_many = len(urls) > 1 or isinstance(effective_config, list)" in sourcedef test_no_crawler_configs_uses_single(self):"""When crawler_configs is None, the original single config path is used."""