Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(deep-crawl): reset page budget on a fresh run of a reused strategy (#2309)

Aryan Ranec24d12d18679
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
index abf53f05..6eb9fd27 100644
--- a/crawl4ai/deep_crawling/bff_strategy.py
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -235,6 +235,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
await queue.put((-initial_score, 0, start_url, None))
visited: Set[str] = set()
depths: Dict[str, int] = {start_url: 0}
+ self._pages_crawled = 0
# Initialize shadow list if callback is set
if self._on_state_change:
self._queue_shadow = [(-initial_score, 0, start_url, None)]
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
index 2b00a6cd..c78c64c0 100644
--- a/crawl4ai/deep_crawling/bfs_strategy.py
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -232,6 +232,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
# current_level holds tuples: (url, parent_url)
current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
depths: Dict[str, int] = {start_url: 0}
+ self._pages_crawled = 0
results: List[CrawlResult] = []
@@ -329,6 +330,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
visited: Set[str] = set()
current_level: List[Tuple[str, Optional[str]]] = [(start_url, None)]
depths: Dict[str, int] = {start_url: 0}
+ self._pages_crawled = 0
while current_level and not self._cancel_event.is_set():
# Check external cancellation callback before processing this level
diff --git a/crawl4ai/deep_crawling/dfs_strategy.py b/crawl4ai/deep_crawling/dfs_strategy.py
index 3e4987f2..10b36657 100644
--- a/crawl4ai/deep_crawling/dfs_strategy.py
+++ b/crawl4ai/deep_crawling/dfs_strategy.py
@@ -60,6 +60,7 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
depths: Dict[str, int] = {start_url: 0}
results: List[CrawlResult] = []
+ self._pages_crawled = 0
self._reset_seen(start_url)
while stack and not self._cancel_event.is_set():
@@ -169,6 +170,7 @@ class DFSDeepCrawlStrategy(BFSDeepCrawlStrategy):
visited: Set[str] = set()
stack: List[Tuple[str, Optional[str], int]] = [(start_url, None, 0)]
depths: Dict[str, int] = {start_url: 0}
+ self._pages_crawled = 0
self._reset_seen(start_url)
while stack and not self._cancel_event.is_set():
diff --git a/tests/deep_crawling/test_deep_crawl_reuse_2309.py b/tests/deep_crawling/test_deep_crawl_reuse_2309.py
new file mode 100644
index 00000000..3756e2c5
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_reuse_2309.py
@@ -0,0 +1,98 @@
+"""
+Regression tests for #2309: reusing a deep crawl strategy must not carry the
+previous run's max_pages budget into a fresh run.
+
+A fresh run (no resume_state) starts its page counter at zero; a resumed run
+still continues from the saved pages_crawled.
+"""
+
+import pytest
+from unittest.mock import MagicMock
+
+from crawl4ai.deep_crawling import (
+ BFSDeepCrawlStrategy,
+ DFSDeepCrawlStrategy,
+ BestFirstCrawlingStrategy,
+)
+
+
+STRATEGIES = [BFSDeepCrawlStrategy, DFSDeepCrawlStrategy, BestFirstCrawlingStrategy]
+
+
+def create_mock_config(stream=False):
+ config = MagicMock()
+ config.stream = stream
+
+ def clone_config(**kwargs):
+ new_config = MagicMock()
+ new_config.stream = kwargs.get("stream", stream)
+ new_config.clone = MagicMock(side_effect=clone_config)
+ return new_config
+
+ config.clone = MagicMock(side_effect=clone_config)
+ return config
+
+
+def create_mock_crawler():
+ """Crawler that succeeds on every URL; each start page links to one child."""
+
+ async def mock_arun_many(urls, config):
+ results = []
+ for url in urls:
+ result = MagicMock()
+ result.url = url
+ result.success = True
+ result.metadata = {}
+ links = [] if url.endswith("/child") else [{"href": f"{url}/child"}]
+ result.links = {"internal": links, "external": []}
+ results.append(result)
+
+ if config.stream:
+ async def gen():
+ for r in results:
+ yield r
+ return gen()
+ return results
+
+ crawler = MagicMock()
+ crawler.arun_many = mock_arun_many
+ return crawler
+
+
+async def run_strategy(strategy, start_url, stream):
+ config = create_mock_config(stream=stream)
+ if stream:
+ gen = await strategy.arun(start_url, create_mock_crawler(), config)
+ return [r async for r in gen]
+ return await strategy.arun(start_url, create_mock_crawler(), config)
+
+
[email protected]("strategy_cls", STRATEGIES)
[email protected]("stream", [False, True])
+async def test_reused_strategy_gets_fresh_page_budget(strategy_cls, stream):
+ # Each run crawls two pages (start + child), below max_pages, so only a
+ # counter leaked from the previous run can cut the second run short.
+ strategy = strategy_cls(max_depth=1, max_pages=3)
+
+ for url in ["https://example.test/first", "https://example.test/second"]:
+ results = await run_strategy(strategy, url, stream)
+ assert sorted(r.url for r in results) == [url, f"{url}/child"]
+
+
[email protected]("stream", [False, True])
+async def test_resumed_run_keeps_saved_page_count(stream):
+ saved_state = {
+ "strategy_type": "bfs",
+ "visited": ["https://example.test/"],
+ "pending": [{"url": "https://example.test/next", "parent_url": "https://example.test/"}],
+ "depths": {"https://example.test/": 0, "https://example.test/next": 1},
+ "pages_crawled": 1,
+ }
+ strategy = BFSDeepCrawlStrategy(max_depth=2, max_pages=1, resume_state=saved_state)
+
+ results = await run_strategy(strategy, "https://example.test/", stream)
+
+ # The saved run already used the whole budget, so nothing more is crawled.
+ assert results == []