Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(deep-crawl): best_first keeps boundary page at max_pages (#859)
unclecodedc5e2a84c9cb
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.pyindex 072ec2f9..81db7611 100644--- a/crawl4ai/deep_crawling/bff_strategy.py+++ b/crawl4ai/deep_crawling/bff_strategy.py@@ -291,13 +291,17 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):# Count only successful crawls toward max_pages limitif result.success:self._pages_crawled += 1- # Check if we've reached the limit during batch processing- if self._pages_crawled >= self.max_pages:- self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")- break # Exit the generator-++ # Yield the result before any limit check so the boundary page is+ # kept (mirrors BFS/DFS, which append the result before breaking).yield result-++ # Stop once the limit is reached, but only after yielding the+ # successful boundary page above.+ if result.success and self._pages_crawled >= self.max_pages:+ self.logger.info(f"Max pages limit ({self.max_pages}) reached during batch, stopping crawl")+ break # Exit the generator+# Only discover links from successful crawlsif result.success:# Discover new links from this resultdiff --git a/tests/deep_crawling/test_deep_crawl_resume.py b/tests/deep_crawling/test_deep_crawl_resume.pyindex 4b41e0fe..82173a96 100644--- a/tests/deep_crawling/test_deep_crawl_resume.py+++ b/tests/deep_crawling/test_deep_crawl_resume.py@@ -747,6 +747,41 @@ class TestBestFirstRegressions:assert strategy.url_scorer is scorer+class TestMaxPagesBoundaryYielded:+ """best_first must YIELD exactly max_pages results (boundary page kept).++ Regression for #859: best_first used to break BEFORE yielding the page that+ hit the limit, so it returned max_pages-1 results while BFS already returned+ max_pages. The downstream scan count (discovered_urls) is derived from the+ yielded results, so the off-by-one surfaced as "11 requested -> 10 found".+ BFS is included as the working reference. (DFS batch mode has a separate,+ unrelated overshoot quirk and is intentionally not asserted here.)+ """++ @pytest.mark.parametrize(+ "strategy_class",+ [BFSDeepCrawlStrategy, BestFirstCrawlingStrategy],+ )+ @pytest.mark.asyncio+ async def test_yields_exactly_max_pages(self, strategy_class):+ max_pages = 11+ strategy = strategy_class(max_depth=10, max_pages=max_pages)++ mock_crawler = create_mock_crawler_unlimited_links()+ # _arun_batch is the path the scan worker takes (crawler.arun with the+ # default stream=False). best_first's _arun_batch re-clones with+ # stream=True internally, so clone must honour the requested flag.+ mock_config = create_mock_config()+ mock_config.clone = lambda **kw: create_mock_config(stream=kw.get("stream", False))++ results = await strategy._arun_batch("https://example.com", mock_crawler, mock_config)++ assert len(results) == max_pages, (+ f"{strategy_class.__name__} yielded {len(results)} results, "+ f"expected exactly max_pages={max_pages}"+ )++class TestAPICompatibility:"""Ensure API/serialization compatibility."""