Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(docker): apply crawler_configs to single-URL and streaming requests

Talel Boussetta81efcc35f00a
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index b03841e7..ab1f1aac 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -648,6 +648,32 @@ async def stream_results(crawler: AsyncWebCrawler, results_gen: AsyncGenerator)
await _dispose_crawler(crawler)
+def _load_crawler_configs(
+ crawler_configs: List[dict],
+ base_config: Optional[dict] = None,
+ stream: bool = False,
+) -> List[CrawlerRunConfig]:
+ """Deserialize a per-URL crawler_configs list under the untrusted gate.
+ Shared by the streaming and non-streaming crawl handlers so both apply the
+ same boundary and wire the PDF URL validator into every entry."""
+ from crawl4ai.processors.pdf import PDFContentScrapingStrategy
+
+ config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
+ for cfg in config_list:
+ for key, value in (base_config or {}).items():
+ if hasattr(cfg, key):
+ current_value = getattr(cfg, key)
+ if current_value is None or current_value == "":
+ setattr(cfg, key, value)
+ # SSRF: per-URL PDF strategies need the validator wired too
+ if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
+ cfg.scraping_strategy.url_validator = validate_url_destination
+ if stream:
+ # arun_many takes the stream flag from the first config of a list.
+ cfg.stream = True
+ return config_list
+
+
def _normalize_and_validate_seeds(urls: List[str]) -> Tuple[List[str], List[CrawlResult]]:
"""Prefix bare hosts with https:// and SSRF-validate every seed URL's
destination. Shared by the streaming and non-streaming crawl handlers so a
@@ -736,19 +762,9 @@ async def handle_crawl_request(
base_config = config["crawler"]["base_config"]
# Build the config(s) to pass to arun/arun_many
- if crawler_configs and len(urls) > 1:
+ if crawler_configs:
# Per-URL config list: deserialize each and apply base_config
- config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
- for cfg in config_list:
- for key, value in base_config.items():
- if hasattr(cfg, key):
- current_value = getattr(cfg, key)
- if current_value is None or current_value == "":
- setattr(cfg, key, value)
- # SSRF: per-URL PDF strategies need the validator wired too
- if isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy):
- cfg.scraping_strategy.url_validator = validate_url_destination
- effective_config = config_list
+ effective_config = _load_crawler_configs(crawler_configs, base_config)
else:
# Single config (original behavior)
for key, value in base_config.items():
@@ -759,9 +775,12 @@ async def handle_crawl_request(
effective_config = crawler_config
results = []
- func = getattr(crawler, "arun" if len(urls) == 1 else "arun_many")
+ # arun takes one config, so a per-URL list goes to arun_many even for
+ # a single URL; arun_many matches each URL against the list.
+ use_many = len(urls) > 1 or isinstance(effective_config, list)
+ func = getattr(crawler, "arun_many" if use_many else "arun")
partial_func = partial(func,
- urls[0] if len(urls) == 1 else urls,
+ urls if use_many else urls[0],
config=effective_config,
dispatcher=dispatcher)
# Optional per-crawl wall-clock deadline (config limits.wall_clock_s; 0 = none).
@@ -911,7 +930,8 @@ async def handle_stream_crawl_request(
browser_config: dict,
crawler_config: dict,
config: dict,
- hooks_config: Optional[dict] = None
+ hooks_config: Optional[dict] = None,
+ crawler_configs: Optional[List[dict]] = None,
) -> Tuple[AsyncWebCrawler, AsyncGenerator, Optional[Dict]]:
"""Handle streaming crawl requests with optional hooks."""
hooks_info = None
@@ -936,6 +956,9 @@ async def handle_stream_crawl_request(
clamp_deep_crawl(crawler_config)
crawler_config.stream = True
+ config_list = (
+ _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None
+ )
# Deep crawl streaming supports exactly one start URL
if crawler_config.deep_crawl_strategy is not None and len(urls) + len(refused) != 1:
@@ -984,7 +1007,7 @@ async def handle_stream_crawl_request(
)
results_gen = await crawler.arun_many(
urls=urls,
- config=crawler_config,
+ config=config_list or crawler_config,
dispatcher=dispatcher
)
diff --git a/deploy/docker/server.py b/deploy/docker/server.py
index 3d71a307..5f08bae4 100644
--- a/deploy/docker/server.py
+++ b/deploy/docker/server.py
@@ -1032,7 +1032,8 @@ async def stream_process(crawl_request: CrawlRequestWithHooks):
browser_config=crawl_request.browser_config,
crawler_config=crawl_request.crawler_config,
config=config,
- hooks_config=hooks_config
+ hooks_config=hooks_config,
+ crawler_configs=crawl_request.crawler_configs,
)
# Add hooks info to response headers if available
diff --git a/deploy/docker/tests/test_crawler_configs_routing.py b/deploy/docker/tests/test_crawler_configs_routing.py
new file mode 100644
index 00000000..b97093df
--- /dev/null
+++ b/deploy/docker/tests/test_crawler_configs_routing.py
@@ -0,0 +1,131 @@
+"""crawler_configs must reach arun_many for one URL and on the streaming path.
+
+The per-URL config list (#1837) was only applied when a non-streaming request
+carried two or more URLs. With one URL the handler called arun(), which takes a
+single config, and /crawl/stream never passed the list down, so both requests
+ran with crawler_config alone and still answered HTTP 200.
+"""
+
+import asyncio
+
+from crawl4ai import CrawlerRunConfig
+from crawl4ai.models import CrawlResult
+
+PER_URL = [
+ {
+ "type": "CrawlerRunConfig",
+ "params": {"url_matcher": "*example.com*", "word_count_threshold": 7},
+ },
+]
+
+
+class RecordingCrawler:
+ def __init__(self):
+ self.calls = []
+
+ async def arun(self, url, config=None, **kwargs):
+ self.calls.append(("arun", url, config))
+ return CrawlResult(url=url, html="", success=True)
+
+ async def arun_many(self, urls, config=None, **kwargs):
+ self.calls.append(("arun_many", urls, config))
+ return [CrawlResult(url=url, html="", success=True) for url in urls]
+
+
+def _install(monkeypatch, crawler):
+ import api
+ import crawler_pool
+
+ async def get_crawler(*args, **kwargs):
+ return crawler
+
+ async def release_crawler(*args, **kwargs):
+ return None
+
+ # Offline: the seed check resolves DNS.
+ monkeypatch.setattr(api, "validate_url_destination", lambda url: None)
+ monkeypatch.setattr(crawler_pool, "get_crawler", get_crawler)
+ monkeypatch.setattr(crawler_pool, "release_crawler", release_crawler)
+ return api
+
+
+def test_single_url_uses_the_config_list(server_module, monkeypatch):
+ crawler = RecordingCrawler()
+ api = _install(monkeypatch, crawler)
+
+ asyncio.run(
+ api.handle_crawl_request(
+ urls=["https://example.com/"],
+ browser_config={},
+ crawler_config={},
+ config=server_module.config,
+ crawler_configs=PER_URL,
+ )
+ )
+
+ [(method, urls, config)] = crawler.calls
+ assert method == "arun_many"
+ assert urls == ["https://example.com/"]
+ assert [c.word_count_threshold for c in config] == [7]
+
+
+def test_single_url_without_a_list_still_uses_arun(server_module, monkeypatch):
+ crawler = RecordingCrawler()
+ api = _install(monkeypatch, crawler)
+
+ asyncio.run(
+ api.handle_crawl_request(
+ urls=["https://example.com/"],
+ browser_config={},
+ crawler_config={},
+ config=server_module.config,
+ )
+ )
+
+ [(method, url, config)] = crawler.calls
+ assert method == "arun"
+ assert url == "https://example.com/"
+ assert isinstance(config, CrawlerRunConfig)
+
+
+def test_stream_passes_the_config_list(server_module, monkeypatch):
+ crawler = RecordingCrawler()
+ api = _install(monkeypatch, crawler)
+
+ asyncio.run(
+ api.handle_stream_crawl_request(
+ urls=["https://example.com/", "https://example.org/"],
+ browser_config={},
+ crawler_config={},
+ config=server_module.config,
+ crawler_configs=PER_URL,
+ )
+ )
+
+ [(method, urls, config)] = crawler.calls
+ assert method == "arun_many"
+ assert [c.word_count_threshold for c in config] == [7]
+ # arun_many takes the stream flag from the first config of a list.
+ assert all(c.stream for c in config)
+
+
+def test_stream_endpoint_forwards_the_list(server_module, monkeypatch):
+ captured = {}
+
+ async def fake_handle_stream(**kwargs):
+ captured.update(kwargs)
+ raise RuntimeError("stop after capture")
+
+ monkeypatch.setattr(
+ server_module, "handle_stream_crawl_request", fake_handle_stream
+ )
+ request = server_module.CrawlRequestWithHooks(
+ urls=["https://example.com/"], crawler_configs=PER_URL
+ )
+
+ try:
+ asyncio.run(server_module.stream_process(crawl_request=request))
+ except RuntimeError:
+ pass
+
+ assert captured["crawler_configs"] == PER_URL
diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py
index 45150e58..4a888e07 100644
--- a/tests/test_issue_1837_config_list.py
+++ b/tests/test_issue_1837_config_list.py
@@ -88,11 +88,11 @@ class TestConfigListLogic:
"""Verify the branching logic for single vs list configs."""
def test_api_uses_config_list_when_provided(self):
- """When crawler_configs is provided with multiple URLs, it should be used."""
+ """When crawler_configs is provided, it should be used, for one URL or many."""
with open("deploy/docker/api.py") as f:
source = f.read()
# Should check crawler_configs and build a list
- assert "if crawler_configs and len(urls) > 1:" in source
+ assert "if crawler_configs:" in source
assert "config_list" in source
def test_api_falls_back_to_single_config(self):
@@ -125,12 +125,14 @@ class TestServerEndpoint:
class TestBackwardCompatibility:
"""Ensure existing single-config requests still work."""
- def test_single_url_ignores_crawler_configs(self):
- """With a single URL, crawler_configs should be ignored (uses arun, not arun_many)."""
+ def test_single_url_with_config_list_uses_arun_many(self):
+ """A config list goes to arun_many even for a single URL (arun takes one config).
+
+ Behavioral coverage: deploy/docker/tests/test_crawler_configs_routing.py.
+ """
with open("deploy/docker/api.py") as f:
source = f.read()
- # Single URL uses arun which only takes one config
- assert '"arun" if len(urls) == 1 else "arun_many"' in source
+ assert "use_many = len(urls) > 1 or isinstance(effective_config, list)" in source
def test_no_crawler_configs_uses_single(self):
"""When crawler_configs is None, the original single config path is used."""