Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(deep-crawl): best_first keeps boundary page at max_pages (#859)

unclecodedc5e2a84c9cb
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
index 072ec2f9..81db7611 100644
--- a/crawl4ai/deep_crawling/bff_strategy.py
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -291,13 +291,17 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
# Count only successful crawls toward max_pages limit
if result.success:
self._pages_crawled += 1
- # Check if we've reached the limit during batch processing
- if self._pages_crawled >= self.max_pages:
- self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
- break # Exit the generator
-
+
+ # Yield the result before any limit check so the boundary page is
+ # kept (mirrors BFS/DFS, which append the result before breaking).
yield result
-
+
+ # Stop once the limit is reached, but only after yielding the
+ # successful boundary page above.
+ if result.success and self._pages_crawled >= self.max_pages:
+ self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")
+ break # Exit the generator
+
# Only discover links from successful crawls
if result.success:
# Discover new links from this result
diff --git a/tests/deep_crawling/test_deep_crawl_resume.py b/tests/deep_crawling/test_deep_crawl_resume.py
index 4b41e0fe..82173a96 100644
--- a/tests/deep_crawling/test_deep_crawl_resume.py
+++ b/tests/deep_crawling/test_deep_crawl_resume.py
@@ -747,6 +747,41 @@ class TestBestFirstRegressions:
assert strategy.url_scorer is scorer
+class TestMaxPagesBoundaryYielded:
+ """best_first must YIELD exactly max_pages results (boundary page kept).
+
+ Regression for #859: best_first used to break BEFORE yielding the page that
+ hit the limit, so it returned max_pages-1 results while BFS already returned
+ max_pages. The downstream scan count (discovered_urls) is derived from the
+ yielded results, so the off-by-one surfaced as "11 requested -> 10 found".
+ BFS is included as the working reference. (DFS batch mode has a separate,
+ unrelated overshoot quirk and is intentionally not asserted here.)
+ """
+
+ @pytest.mark.parametrize(
+ "strategy_class",
+ [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy],
+ )
+ @pytest.mark.asyncio
+ async def test_yields_exactly_max_pages(self, strategy_class):
+ max_pages = 11
+ strategy = strategy_class(max_depth=10, max_pages=max_pages)
+
+ mock_crawler = create_mock_crawler_unlimited_links()
+ # _arun_batch is the path the scan worker takes (crawler.arun with the
+ # default stream=False). best_first's _arun_batch re-clones with
+ # stream=True internally, so clone must honour the requested flag.
+ mock_config = create_mock_config()
+ mock_config.clone = lambda **kw: create_mock_config(stream=kw.get("stream", False))
+
+ results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)
+
+ assert len(results) == max_pages, (
+ f"{strategy_class.__name__} yielded {len(results)} results, "
+ f"expected exactly max_pages={max_pages}"
+ )
+
+
class TestAPICompatibility:
"""Ensure API/serialization compatibility."""