Skip to content

Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.

fix(deep-crawl): drop the O(n^2) parent scan and the duplicate best-first enqueue

ntohidi19f3e5e11f33
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.py
index 511fde69..4bc247c0 100644
--- a/crawl4ai/deep_crawling/bff_strategy.py
+++ b/crawl4ai/deep_crawling/bff_strategy.py
@@ -183,6 +183,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
self.stats.urls_skipped += 1
continue
+ visited.add(base_url)
valid_links.append(base_url)
# Record the new depths and add to next_links
@@ -214,6 +215,10 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
self._pages_crawled = self._resume_state.get("pages_crawled", 0)
# Restore queue from saved items
queue_items = self._resume_state.get("queue_items", [])
+ # Older checkpoints could hold a URL twice and recorded only crawled
+ # URLs in "visited"; drop the repeats and treat everything queued as seen.
+ queue_items = list({item["url"]: item for item in queue_items}.values())
+ visited.update(item["url"] for item in queue_items)
for item in queue_items:
await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))
# Initialize shadow list if callback is set
@@ -226,7 +231,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
# Original initialization
initial_score = self.url_scorer.score(start_url) if self.url_scorer else 0
await queue.put((-initial_score, 0, start_url, None))
- visited: Set[str] = set()
+ visited: Set[str] = {start_url}
depths: Dict[str, int] = {start_url: 0}
# Initialize shadow list if callback is set
if self._on_state_change:
@@ -263,10 +268,6 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):
self._queue_shadow.remove(item)
except ValueError:
pass # Item may have been removed already
- score, depth, url, parent_url = item
- if url in visited:
- continue
- visited.add(url)
batch.append(item)
if not batch:
diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.py
index dfb75927..2b00a6cd 100644
--- a/crawl4ai/deep_crawling/bfs_strategy.py
+++ b/crawl4ai/deep_crawling/bfs_strategy.py
@@ -248,6 +248,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
next_level: List[Tuple[str, Optional[str]]] = []
urls = [url for url, _ in current_level]
+ # reversed() keeps first-parent-wins if a resumed level repeats a URL
+ parents = dict(reversed(current_level))
# Clone the config to disable deep crawling recursion and enforce batch mode.
batch_config = config.clone(deep_crawl_strategy=None, stream=False)
@@ -258,7 +260,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
depth = depths.get(url, 0)
result.metadata = result.metadata or {}
result.metadata["depth"] = depth
- parent_url = next((parent for (u, parent) in current_level if u == url), None)
+ parent_url = parents.get(url)
result.metadata["parent_url"] = parent_url
results.append(result)
@@ -336,6 +338,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
next_level: List[Tuple[str, Optional[str]]] = []
urls = [url for url, _ in current_level]
+ # reversed() keeps first-parent-wins if a resumed level repeats a URL
+ parents = dict(reversed(current_level))
visited.update(urls)
stream_config = config.clone(deep_crawl_strategy=None, stream=True)
@@ -348,7 +352,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):
depth = depths.get(url, 0)
result.metadata = result.metadata or {}
result.metadata["depth"] = depth
- parent_url = next((parent for (u, parent) in current_level if u == url), None)
+ parent_url = parents.get(url)
result.metadata["parent_url"] = parent_url
# Count only successful crawls
diff --git a/tests/deep_crawling/test_deep_crawl_perf_2242.py b/tests/deep_crawling/test_deep_crawl_perf_2242.py
new file mode 100644
index 00000000..f2782676
--- /dev/null
+++ b/tests/deep_crawling/test_deep_crawl_perf_2242.py
@@ -0,0 +1,147 @@
+"""Regression tests for issue #2242.
+
+1. BFS matched a fetched result back to its parent by re-scanning the whole
+ level, which is O(n^2) per level.
+2. Best-First only marked a URL visited when it was dequeued, so two pages
+ linking to the same third page pushed it onto the priority queue twice.
+"""
+
+import asyncio
+from collections import Counter
+from typing import Any, Dict, List
+from unittest.mock import MagicMock
+
+import pytest
+
+from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy
+
+
+def make_config(stream: bool = False):
+ config = MagicMock()
+ config.clone = MagicMock(return_value=config)
+ config.stream = stream
+ return config
+
+
+class FakeResult:
+ """Cheap stand-in for CrawlResult (MagicMock is too slow to time against)."""
+
+ def __init__(self, url: str, children: List[str]):
+ self.url = url
+ self.success = True
+ self.metadata: Dict[str, Any] = {}
+ self.links = {
+ "internal": [{"href": child} for child in children],
+ "external": [],
+ }
+
+
+def make_crawler(link_map: Dict[str, List[str]]):
+ """Mock crawler serving a fixed url -> child urls map."""
+
+ async def arun_many(urls, config):
+ results = [FakeResult(url, link_map.get(url, [])) for url in urls]
+
+ if config.stream:
+ async def gen():
+ for result in results:
+ yield result
+ return gen()
+ return results
+
+ crawler = MagicMock()
+ crawler.arun_many = arun_many
+ return crawler
+
+
+ROOT = "https://example.com"
+
+
[email protected]("stream", [False, True])
+async def test_bfs_parent_url_is_correct_for_every_child(stream):
+ """Parent bookkeeping must stay correct now that it uses a lookup table."""
+ children = [f"{ROOT}/p{i}" for i in range(50)]
+ link_map = {ROOT: children, **{c: [f"{c}/leaf"] for c in children}}
+
+ strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=100)
+ crawler, config = make_crawler(link_map), make_config(stream=stream)
+
+ if stream:
+ results = [r async for r in strategy._arun_stream(ROOT, crawler, config)]
+ else:
+ results = await strategy._arun_batch(ROOT, crawler, config)
+
+ parents = {r.url: r.metadata["parent_url"] for r in results}
+ assert parents[ROOT] is None
+ assert len(results) == len(children) + 1
+ for child in children:
+ assert parents[child] == ROOT
+
+
+async def test_bfs_level_bookkeeping_scales_linearly():
+ """Matching results back to parents must not re-scan the level per result.
+
+ max_depth=0 keeps link discovery out of the loop and resume_state seeds a
+ level of arbitrary width, so the only per-result work left is the parent
+ lookup. Quadratic bookkeeping shows up as a ~4x cost for 2x the URLs.
+ """
+
+ async def run(n: int) -> float:
+ urls = [f"{ROOT}/p{i}" for i in range(n)]
+ strategy = BFSDeepCrawlStrategy(
+ max_depth=0,
+ max_pages=n + 1,
+ resume_state={
+ "visited": urls,
+ "pending": [{"url": u, "parent_url": ROOT} for u in urls],
+ "depths": {u: 1 for u in urls},
+ "pages_crawled": 0,
+ },
+ )
+ crawler, config = make_crawler({}), make_config()
+ loop = asyncio.get_running_loop()
+ start = loop.time()
+ results = await strategy._arun_batch(ROOT, crawler, config)
+ elapsed = loop.time() - start
+ assert len(results) == n
+ return elapsed
+
+ async def best_of(n: int, rounds: int = 3) -> float:
+ return min([await run(n) for _ in range(rounds)])
+
+ await run(200) # warm up
+ small = await best_of(2000)
+ large = await best_of(4000)
+
+ # 2x the URLs: linear is ~2x, the old full-level scan was ~4x.
+ assert large / small < 3, f"level bookkeeping looks superlinear: {large / small:.1f}x"
+
+
+async def test_best_first_queues_a_shared_url_once():
+ """Two parents linking to the same page must enqueue it a single time."""
+ link_map = {
+ ROOT: [f"{ROOT}/a", f"{ROOT}/b"],
+ f"{ROOT}/a": [f"{ROOT}/shared"],
+ f"{ROOT}/b": [f"{ROOT}/shared"],
+ f"{ROOT}/shared": [],
+ }
+
+ snapshots: List[List[str]] = []
+
+ async def on_state_change(state: Dict[str, Any]):
+ snapshots.append([item["url"] for item in state["queue_items"]])
+
+ strategy = BestFirstCrawlingStrategy(
+ max_depth=2, max_pages=10, on_state_change=on_state_change
+ )
+ crawled = [
+ r.url
+ async for r in strategy._arun_stream(ROOT, make_crawler(link_map), make_config(stream=True))
+ ]
+
+ for queue in snapshots:
+ assert not [u for u, n in Counter(queue).items() if n > 1], f"duplicate in queue: {queue}"
+ assert sorted(crawled) == sorted(link_map)