Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

style(docker): black-format the new crawler_configs code

Talel Boussetta4d645a41334d
diff --git a/deploy/docker/api.py b/deploy/docker/api.py
index 9259efee..d9ce3ade 100644
--- a/deploy/docker/api.py
+++ b/deploy/docker/api.py
@@ -658,7 +658,10 @@ def _load_crawler_configs(
same boundary and wire the PDF URL validator into every entry."""
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
- config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]
+ config_list = [
+ CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED)
+ for cc in crawler_configs
+ ]
for cfg in config_list:
for key, value in (base_config or {}).items():
if hasattr(cfg, key):
@@ -685,7 +688,9 @@ def _needs_pdf_crawler(
from crawl4ai.processors.pdf import PDFContentScrapingStrategy
configs = config_list or [crawler_config]
- pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs]
+ pdf = [
+ isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs
+ ]
if any(pdf) and not all(pdf):
raise HTTPException(
status_code=400,
@@ -761,7 +766,11 @@ async def handle_crawl_request(
base_config = config["crawler"]["base_config"]
# Per-URL config list: deserialize each and apply base_config. Loaded
# before a crawler is chosen, because the list decides which one runs.
- config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None
+ config_list = (
+ _load_crawler_configs(crawler_configs, base_config)
+ if crawler_configs
+ else None
+ )
from crawler_pool import get_crawler, release_crawler
from crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy
@@ -980,7 +989,9 @@ async def handle_stream_crawl_request(
clamp_deep_crawl(crawler_config)
crawler_config.stream = True
config_list = (
- _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None
+ _load_crawler_configs(crawler_configs, stream=True)
+ if crawler_configs
+ else None
)
# Deep crawl streaming supports exactly one start URL
diff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.py
index 0e5463bb..0e017be9 100644
--- a/tests/test_issue_1837_config_list.py
+++ b/tests/test_issue_1837_config_list.py
@@ -92,7 +92,7 @@ class TestConfigListLogic:
with open("deploy/docker/api.py") as f:
source = f.read()
# Should check crawler_configs and build a list
- assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in source
+ assert "_load_crawler_configs(crawler_configs, base_config)" in source
assert "config_list" in source
def test_api_falls_back_to_single_config(self):