Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
style(docker): black-format the new crawler_configs code
Talel Boussetta4d645a41334d
diff --git a/deploy/docker/api.py b/deploy/docker/api.pyindex 9259efee..d9ce3ade 100644--- a/deploy/docker/api.py+++ b/deploy/docker/api.py@@ -658,7 +658,10 @@ def _load_crawler_configs(same boundary and wire the PDF URL validator into every entry."""from crawl4ai.processors.pdf import PDFContentScrapingStrategy- config_list = [CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED) for cc in crawler_configs]+ config_list = [+ CrawlerRunConfig.load(cc, provenance=Provenance.UNTRUSTED)+ for cc in crawler_configs+ ]for cfg in config_list:for key, value in (base_config or {}).items():if hasattr(cfg, key):@@ -685,7 +688,9 @@ def _needs_pdf_crawler(from crawl4ai.processors.pdf import PDFContentScrapingStrategyconfigs = config_list or [crawler_config]- pdf = [isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs]+ pdf = [+ isinstance(cfg.scraping_strategy, PDFContentScrapingStrategy) for cfg in configs+ ]if any(pdf) and not all(pdf):raise HTTPException(status_code=400,@@ -761,7 +766,11 @@ async def handle_crawl_request(base_config = config["crawler"]["base_config"]# Per-URL config list: deserialize each and apply base_config. Loaded# before a crawler is chosen, because the list decides which one runs.- config_list = _load_crawler_configs(crawler_configs, base_config) if crawler_configs else None+ config_list = (+ _load_crawler_configs(crawler_configs, base_config)+ if crawler_configs+ else None+ )from crawler_pool import get_crawler, release_crawlerfrom crawl4ai.processors.pdf import PDFContentScrapingStrategy, PDFCrawlerStrategy@@ -980,7 +989,9 @@ async def handle_stream_crawl_request(clamp_deep_crawl(crawler_config)crawler_config.stream = Trueconfig_list = (- _load_crawler_configs(crawler_configs, stream=True) if crawler_configs else None+ _load_crawler_configs(crawler_configs, stream=True)+ if crawler_configs+ else None)# Deep crawl streaming supports exactly one start URLdiff --git a/tests/test_issue_1837_config_list.py b/tests/test_issue_1837_config_list.pyindex 0e5463bb..0e017be9 100644--- a/tests/test_issue_1837_config_list.py+++ b/tests/test_issue_1837_config_list.py@@ -92,7 +92,7 @@ class TestConfigListLogic:with open("deploy/docker/api.py") as f:source = f.read()# Should check crawler_configs and build a list- assert "_load_crawler_configs(crawler_configs, base_config) if crawler_configs else None" in source+ assert "_load_crawler_configs(crawler_configs, base_config)" in sourceassert "config_list" in sourcedef test_api_falls_back_to_single_config(self):