Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(deep-crawl): cap each BFS level to the remaining max_pages budget

drakeo3387e76f2090418
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
index 8353038c..c62e6363 100644
--- a/crawl4ai/deep_crawling/bfs_strategy.py
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -247,6 +247,12 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
break
next_level: List[Tuple[str, Optional[str]]] = []
+ # Siblings in a level queue links before the level's own pages are all
+ # counted, so cap the level to the budget left before fetching it.
+ # max_pages defaults to infinity, which is not a valid slice index.
+ remaining = self.max_pages - self._pages_crawled
+ if remaining < len(current_level):
+ current_level = current_level[:max(int(remaining), 0)]
urls = [url for url, _ in current_level]
# reversed() keeps first-parent-wins if a resumed level repeats a URL
parents = dict(reversed(current_level))
@@ -337,6 +343,15 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
break
next_level: List[Tuple[str, Optional[str]]] = []
+ # Siblings in a level queue links before the level's own pages are all
+ # counted, so cap the level to the budget left before fetching it.
+ # max_pages defaults to infinity, which is not a valid slice index.
+ remaining = self.max_pages - self._pages_crawled
+ if remaining < len(current_level):
+ current_level = current_level[:max(int(remaining), 0)]
+ if not current_level:
+ self.logger.info(f"Max pages limit ({self.max_pages}) reached, stopping crawl")
+ break
urls = [url for url, _ in current_level]
# reversed() keeps first-parent-wins if a resumed level repeats a URL
parents = dict(reversed(current_level))
diff --git a/tests/deep_crawling/test_bfs_max_pages_next_level.py b/tests/deep_crawling/test_bfs_max_pages_next_level.py
index edd6497d..973b7852 100644
--- a/tests/deep_crawling/test_bfs_max_pages_next_level.py
+++ b/tests/deep_crawling/test_bfs_max_pages_next_level.py
@@ -1,7 +1,10 @@
-"""max_pages must count URLs already queued for the next level (#2338)."""
+"""BFS deep crawl must not fetch more than max_pages pages (#2338)."""
+
+from collections import Counter
import pytest
+from crawl4ai import CrawlerRunConfig
from crawl4ai.deep_crawling import BFSDeepCrawlStrategy
from crawl4ai.models import CrawlResult
@@ -28,3 +31,81 @@ async def test_link_discovery_budget_counts_next_level():
)
assert len(next_level) == 14
+
+
+# A home page that links to 35 pages, each of which links to 30 more.
+ROOT = "https://example.com/"
+
+
+def _tree_links(url):
+ if url == ROOT:
+ return [f"https://example.com/p{i}" for i in range(35)]
+ if url.count("/") == 3:
+ return [f"{url}/c{j}" for j in range(30)]
+ return []
+
+
+# Every page links to a shared 10-link menu plus 3 pages of its own.
+MENU = [f"https://example.com/m{i}" for i in range(10)]
+
+
+def _menu_links(url):
+ return MENU + [f"{url.rstrip('/')}/n{j}" for j in range(3)]
+
+
+class FakeCrawler:
+ def __init__(self, links=_tree_links):
+ self.links = links
+ self.fetched = 0
+
+ def _results(self, urls):
+ self.fetched += len(urls)
+ return [_result(u, self.links(u)) for u in urls]
+
+ async def arun_many(self, urls, config):
+ results = self._results(urls)
+ if not config.stream:
+ return results
+
+ async def gen():
+ for r in results:
+ yield r
+
+ return gen()
+
+
+async def test_batch_crawl_stops_at_max_pages():
+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)
+ crawler = FakeCrawler()
+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
+ assert crawler.fetched == 50
+ assert len(results) == 50
+ depths = Counter(r.metadata["depth"] for r in results)
+ assert depths == {0: 1, 1: 35, 2: 14}
+
+
+async def test_batch_crawl_stops_at_max_pages_shared_menu():
+ strategy = BFSDeepCrawlStrategy(max_depth=3, max_pages=50)
+ crawler = FakeCrawler(_menu_links)
+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
+ assert crawler.fetched == 50
+ assert len(results) == 50
+
+
+async def test_stream_crawl_fetches_at_most_max_pages():
+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=50)
+ crawler = FakeCrawler()
+ config = CrawlerRunConfig(stream=True)
+ [r async for r in strategy._arun_stream(ROOT, crawler, config)]
+ assert crawler.fetched == 50
+
+
+async def test_batch_crawl_without_max_pages_crawls_everything():
+ strategy = BFSDeepCrawlStrategy(max_depth=2)
+ crawler = FakeCrawler()
+ results = await strategy._arun_batch(ROOT, crawler, CrawlerRunConfig())
+ assert len(results) == 1 + 35 + 35 * 30