Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(deep-crawl): cap each BFS level to the remaining max_pages budget
drakeo3387e76f2090418
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.pyindex 8353038c..c62e6363 100644--- a/crawl4ai/deep_crawling/bfs_strategy.py+++ b/crawl4ai/deep_crawling/bfs_strategy.py@@ -247,6 +247,12 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):breaknext_level: List[Tuple[str, Optional[str]]] = []+ # Siblings in a level queue links before the level's own pages are all+ # counted, so cap the level to the budget left before fetching it.+ # max_pages defaults to infinity, which is not a valid slice index.+ remaining = self.max_pages - self._pages_crawled+ if remaining < len(current_level):+ current_level = current_level[:max(int(remaining), 0)]urls = [url for url, _ in current_level]# reversed() keeps first-parent-wins if a resumed level repeats a URLparents = dict(reversed(current_level))@@ -337,6 +343,15 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):breaknext_level: List[Tuple[str, Optional[str]]] = []+ # Siblings in a level queue links before the level's own pages are all+ # counted, so cap the level to the budget left before fetching it.+ # max_pages defaults to infinity, which is not a valid slice index.+ remaining = self.max_pages - self._pages_crawled+ if remaining < len(current_level):+ current_level = current_level[:max(int(remaining), 0)]+ if not current_level:+ self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")+ breakurls = [url for url, _ in current_level]# reversed() keeps first-parent-wins if a resumed level repeats a URLparents = dict(reversed(current_level))diff --git a/tests/deep_crawling/test_bfs_max_pages_next_level.py b/tests/deep_crawling/test_bfs_max_pages_next_level.pyindex edd6497d..973b7852 100644--- a/tests/deep_crawling/test_bfs_max_pages_next_level.py+++ b/tests/deep_crawling/test_bfs_max_pages_next_level.py@@ -1,7 +1,10 @@-"""max_pages must count URLs already queued for the next level (#2338)."""+"""BFS deep crawl must not fetch more than max_pages pages (#2338)."""++from collections import Counterimport pytest+from crawl4ai import CrawlerRunConfigfrom crawl4ai.deep_crawling import BFSDeepCrawlStrategyfrom crawl4ai.models import CrawlResult@@ -28,3 +31,81 @@ async def test_link_discovery_budget_counts_next_level():)assert len(next_level) == 14+++# A home page that links to 35 pages, each of which links to 30 more.+ROOT = "https://example.com/"+++def _tree_links(url):+ if url == ROOT:+ return [f"https://example.com/p{i}" for i in range(35)]+ if url.count("/") == 3:+ return [f"{url}/c{j}" for j in range(30)]+ return []+++# Every page links to a shared 10-link menu plus 3 pages of its own.+MENU = [f"https://example.com/m{i}" for i in range(10)]+++def _menu_links(url):+ return MENU + [f"{url.rstrip('/')}/n{j}" for j in range(3)]+++class FakeCrawler:+ def __init__(self, links=_tree_links):+ self.links = links+ self.fetched = 0++ def _results(self, urls):+ self.fetched += len(urls)+ return [_result(u, self.links(u)) for u in urls]++ async def arun_many(self, urls, config):+ results = self._results(urls)+ if not config.stream:+ return results++ async def gen():+ for r in results:+ yield r++ return gen()+++async def test_batch_crawl_stops_at_max_pages():+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)+ crawler = FakeCrawler()+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())+ assert crawler.fetched == 50+ assert len(results) == 50+ depths = Counter(r.metadata["depth"] for r in results)+ assert depths == {0: 1, 1: 35, 2: 14}+++async def test_batch_crawl_stops_at_max_pages_shared_menu():+ strategy = BFSDeepCrawlStrategy(max_depth=3, max_pages=50)+ crawler = FakeCrawler(_menu_links)+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())+ assert crawler.fetched == 50+ assert len(results) == 50+++async def test_stream_crawl_fetches_at_most_max_pages():+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)+ crawler = FakeCrawler()+ config = CrawlerRunConfig(stream=True)+ [r async for r in strategy._arun_stream(ROOT, crawler, config)]+ assert crawler.fetched == 50+++async def test_batch_crawl_without_max_pages_crawls_everything():+ strategy = BFSDeepCrawlStrategy(max_depth=2)+ crawler = FakeCrawler()+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())+ assert len(results) == 1 + 35 + 35 * 30