Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(deep-crawl): reset page budget on a fresh run of a reused strategy (#2309)
Aryan Ranec24d12d18679
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.pyindex abf53f05..6eb9fd27 100644--- a/crawl4ai/deep_crawling/bff_strategy.py+++ b/crawl4ai/deep_crawling/bff_strategy.py@@ -235,6 +235,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):await queue.put((-initial_score, 0, start_url, None))visited: Set[str] = set()depths: Dict[str, int] = {start_url: 0}+ self._pages_crawled = 0# Initialize shadow list if callback is setif self._on_state_change:self._queue_shadow = [(-initial_score, 0, start_url, None)]diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.pyindex 2b00a6cd..c78c64c0 100644--- a/crawl4ai/deep_crawling/bfs_strategy.py+++ b/crawl4ai/deep_crawling/bfs_strategy.py@@ -232,6 +232,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):# current_level holds tuples: (url, parent_url)current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]depths: Dict[str, int] = {start_url: 0}+ self._pages_crawled = 0results: List[CrawlResult] = []@@ -329,6 +330,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):visited: Set[str] = set()current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]depths: Dict[str, int] = {start_url: 0}+ self._pages_crawled = 0while current_level and not self._cancel_event.is_set():# Check external cancellation callback before processing this leveldiff --git a/crawl4ai/deep_crawling/dfs_strategy.py b/crawl4ai/deep_crawling/dfs_strategy.pyindex 3e4987f2..10b36657 100644--- a/crawl4ai/deep_crawling/dfs_strategy.py+++ b/crawl4ai/deep_crawling/dfs_strategy.py@@ -60,6 +60,7 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]depths: Dict[str, int] = {start_url: 0}results: List[CrawlResult] = []+ self._pages_crawled = 0self._reset_seen(start_url)while stack and not self._cancel_event.is_set():@@ -169,6 +170,7 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):visited: Set[str] = set()stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]depths: Dict[str, int] = {start_url: 0}+ self._pages_crawled = 0self._reset_seen(start_url)while stack and not self._cancel_event.is_set():diff --git a/tests/deep_crawling/test_deep_crawl_reuse_2309.py b/tests/deep_crawling/test_deep_crawl_reuse_2309.pynew file mode 100644index 00000000..3756e2c5--- /dev/null+++ b/tests/deep_crawling/test_deep_crawl_reuse_2309.py@@ -0,0 +1,98 @@+"""+Regression tests for #2309: reusing a deep crawl strategy must not carry the+previous run's max_pages budget into a fresh run.++A fresh run (no resume_state) starts its page counter at zero; a resumed run+still continues from the saved pages_crawled.+"""++import pytest+from unittest.mock import MagicMock++from crawl4ai.deep_crawling import (+ BFSDeepCrawlStrategy,+ DFSDeepCrawlStrategy,+ BestFirstCrawlingStrategy,+)+++STRATEGIES = [BFSDeepCrawlStrategy, DFSDeepCrawlStrategy, BestFirstCrawlingStrategy]+++def create_mock_config(stream=False):+ config = MagicMock()+ config.stream = stream++ def clone_config(**kwargs):+ new_config = MagicMock()+ new_config.stream = kwargs.get("stream", stream)+ new_config.clone = MagicMock(side_effect=clone_config)+ return new_config++ config.clone = MagicMock(side_effect=clone_config)+ return config+++def create_mock_crawler():+ """Crawler that succeeds on every URL; each start page links to one child."""++ async def mock_arun_many(urls, config):+ results = []+ for url in urls:+ result = MagicMock()+ result.url = url+ result.success = True+ result.metadata = {}+ links = [] if url.endswith("/child") else [{"href": f"{url}/child"}]+ result.links = {"internal": links, "external": []}+ results.append(result)++ if config.stream:+ async def gen():+ for r in results:+ yield r+ return gen()+ return results++ crawler = MagicMock()+ crawler.arun_many = mock_arun_many+ return crawler+++async def run_strategy(strategy, start_url, stream):+ config = create_mock_config(stream=stream)+ if stream:+ gen = await strategy.arun(start_url, create_mock_crawler(), config)+ return [r async for r in gen]+ return await strategy.arun(start_url, create_mock_crawler(), config)++[email protected]("strategy_cls", STRATEGIES)[email protected]("stream", [False, True])+async def test_reused_strategy_gets_fresh_page_budget(strategy_cls, stream):+ # Each run crawls two pages (start + child), below max_pages, so only a+ # counter leaked from the previous run can cut the second run short.+ strategy = strategy_cls(max_depth=1, max_pages=3)++ for url in ["https://example.test/first", "https://example.test/second"]:+ results = await run_strategy(strategy, url, stream)+ assert sorted(r.url for r in results) == [url, f"{url}/child"]++[email protected]("stream", [False, True])+async def test_resumed_run_keeps_saved_page_count(stream):+ saved_state = {+ "strategy_type": "bfs",+ "visited": ["https://example.test/"],+ "pending": [{"url": "https://example.test/next", "parent_url": "https://example.test/"}],+ "depths": {"https://example.test/": 0, "https://example.test/next": 1},+ "pages_crawled": 1,+ }+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=1, resume_state=saved_state)++ results = await run_strategy(strategy, "https://example.test/", stream)++ # The saved run already used the whole budget, so nothing more is crawled.+ assert results == []