Official Crawl4AI Cloud: a hosted web scraping, extraction, and search API for AI agents. Free API key, 1,000 credits a month, no card. Fully open source: this repo is the Apache-2.0 crawler that powers it, and you can self-host it.
Python aiai-agentscrawl4ai-cloudcrawlerdata-extractionllmmarkdownmcpopen-sourceplaywrightpythonragscraperweb-crawlerweb-crawlingweb-scraping
fix(deep-crawl): drop the O(n^2) parent scan and the duplicate best-first enqueue
ntohidi19f3e5e11f33
diff --git a/crawl4ai/deep_crawling/bff_strategy.py b/crawl4ai/deep_crawling/bff_strategy.pyindex 511fde69..4bc247c0 100644--- a/crawl4ai/deep_crawling/bff_strategy.py+++ b/crawl4ai/deep_crawling/bff_strategy.py@@ -183,6 +183,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):self.stats.urls_skipped += 1continue+ visited.add(base_url)valid_links.append(base_url)# Record the new depths and add to next_links@@ -214,6 +215,10 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):self._pages_crawled = self._resume_state.get("pages_crawled", 0)# Restore queue from saved itemsqueue_items = self._resume_state.get("queue_items", [])+ # Older checkpoints could hold a URL twice and recorded only crawled+ # URLs in "visited"; drop the repeats and treat everything queued as seen.+ queue_items = list({item["url"]: item for item in queue_items}.values())+ visited.update(item["url"] for item in queue_items)for item in queue_items:await queue.put((item["score"], item["depth"], item["url"], item["parent_url"]))# Initialize shadow list if callback is set@@ -226,7 +231,7 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):# Original initializationinitial_score = self.url_scorer.score(start_url) if self.url_scorer else 0await queue.put((-initial_score, 0, start_url, None))- visited: Set[str] = set()+ visited: Set[str] = {start_url}depths: Dict[str, int] = {start_url: 0}# Initialize shadow list if callback is setif self._on_state_change:@@ -263,10 +268,6 @@ class BestFirstCrawlingStrategy(DeepCrawlStrategy):self._queue_shadow.remove(item)except ValueError:pass # Item may have been removed already- score, depth, url, parent_url = item- if url in visited:- continue- visited.add(url)batch.append(item)if not batch:diff --git a/crawl4ai/deep_crawling/bfs_strategy.py b/crawl4ai/deep_crawling/bfs_strategy.pyindex dfb75927..2b00a6cd 100644--- a/crawl4ai/deep_crawling/bfs_strategy.py+++ b/crawl4ai/deep_crawling/bfs_strategy.py@@ -248,6 +248,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):next_level: List[Tuple[str, Optional[str]]] = []urls = [url for url, _ in current_level]+ # reversed() keeps first-parent-wins if a resumed level repeats a URL+ parents = dict(reversed(current_level))# Clone the config to disable deep crawling recursion and enforce batch mode.batch_config = config.clone(deep_crawl_strategy=None, stream=False)@@ -258,7 +260,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):depth = depths.get(url, 0)result.metadata = result.metadata or {}result.metadata["depth"] = depth- parent_url = next((parent for (u, parent) in current_level if u == url), None)+ parent_url = parents.get(url)result.metadata["parent_url"] = parent_urlresults.append(result)@@ -336,6 +338,8 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):next_level: List[Tuple[str, Optional[str]]] = []urls = [url for url, _ in current_level]+ # reversed() keeps first-parent-wins if a resumed level repeats a URL+ parents = dict(reversed(current_level))visited.update(urls)stream_config = config.clone(deep_crawl_strategy=None, stream=True)@@ -348,7 +352,7 @@ class BFSDeepCrawlStrategy(DeepCrawlStrategy):depth = depths.get(url, 0)result.metadata = result.metadata or {}result.metadata["depth"] = depth- parent_url = next((parent for (u, parent) in current_level if u == url), None)+ parent_url = parents.get(url)result.metadata["parent_url"] = parent_url# Count only successful crawlsdiff --git a/tests/deep_crawling/test_deep_crawl_perf_2242.py b/tests/deep_crawling/test_deep_crawl_perf_2242.pynew file mode 100644index 00000000..f2782676--- /dev/null+++ b/tests/deep_crawling/test_deep_crawl_perf_2242.py@@ -0,0 +1,147 @@+"""Regression tests for issue #2242.++1. BFS matched a fetched result back to its parent by re-scanning the whole+ level, which is O(n^2) per level.+2. Best-First only marked a URL visited when it was dequeued, so two pages+ linking to the same third page pushed it onto the priority queue twice.+"""++import asyncio+from collections import Counter+from typing import Any, Dict, List+from unittest.mock import MagicMock++import pytest++from crawl4ai.deep_crawling import BFSDeepCrawlStrategy, BestFirstCrawlingStrategy+++def make_config(stream: bool = False):+ config = MagicMock()+ config.clone = MagicMock(return_value=config)+ config.stream = stream+ return config+++class FakeResult:+ """Cheap stand-in for CrawlResult (MagicMock is too slow to time against)."""++ def __init__(self, url: str, children: List[str]):+ self.url = url+ self.success = True+ self.metadata: Dict[str, Any] = {}+ self.links = {+ "internal": [{"href": child} for child in children],+ "external": [],+ }+++def make_crawler(link_map: Dict[str, List[str]]):+ """Mock crawler serving a fixed url -> child urls map."""++ async def arun_many(urls, config):+ results = [FakeResult(url, link_map.get(url, [])) for url in urls]++ if config.stream:+ async def gen():+ for result in results:+ yield result+ return gen()+ return results++ crawler = MagicMock()+ crawler.arun_many = arun_many+ return crawler+++ROOT = "https://example.com"++[email protected]("stream", [False, True])+async def test_bfs_parent_url_is_correct_for_every_child(stream):+ """Parent bookkeeping must stay correct now that it uses a lookup table."""+ children = [f"{ROOT}/p{i}" for i in range(50)]+ link_map = {ROOT: children, **{c: [f"{c}/leaf"] for c in children}}++ strategy = BFSDeepCrawlStrategy(max_depth=1, max_pages=100)+ crawler, config = make_crawler(link_map), make_config(stream=stream)++ if stream:+ results = [r async for r in strategy._arun_stream(ROOT, crawler, config)]+ else:+ results = await strategy._arun_batch(ROOT, crawler, config)++ parents = {r.url: r.metadata["parent_url"] for r in results}+ assert parents[ROOT] is None+ assert len(results) == len(children) + 1+ for child in children:+ assert parents[child] == ROOT+++async def test_bfs_level_bookkeeping_scales_linearly():+ """Matching results back to parents must not re-scan the level per result.++ max_depth=0 keeps link discovery out of the loop and resume_state seeds a+ level of arbitrary width, so the only per-result work left is the parent+ lookup. Quadratic bookkeeping shows up as a ~4x cost for 2x the URLs.+ """++ async def run(n: int) -> float:+ urls = [f"{ROOT}/p{i}" for i in range(n)]+ strategy = BFSDeepCrawlStrategy(+ max_depth=0,+ max_pages=n + 1,+ resume_state={+ "visited": urls,+ "pending": [{"url": u, "parent_url": ROOT} for u in urls],+ "depths": {u: 1 for u in urls},+ "pages_crawled": 0,+ },+ )+ crawler, config = make_crawler({}), make_config()+ loop = asyncio.get_running_loop()+ start = loop.time()+ results = await strategy._arun_batch(ROOT, crawler, config)+ elapsed = loop.time() - start+ assert len(results) == n+ return elapsed++ async def best_of(n: int, rounds: int = 3) -> float:+ return min([await run(n) for _ in range(rounds)])++ await run(200) # warm up+ small = await best_of(2000)+ large = await best_of(4000)++ # 2x the URLs: linear is ~2x, the old full-level scan was ~4x.+ assert large / small < 3, f"level bookkeeping looks superlinear: {large / small:.1f}x"+++async def test_best_first_queues_a_shared_url_once():+ """Two parents linking to the same page must enqueue it a single time."""+ link_map = {+ ROOT: [f"{ROOT}/a", f"{ROOT}/b"],+ f"{ROOT}/a": [f"{ROOT}/shared"],+ f"{ROOT}/b": [f"{ROOT}/shared"],+ f"{ROOT}/shared": [],+ }++ snapshots: List[List[str]] = []++ async def on_state_change(state: Dict[str, Any]):+ snapshots.append([item["url"] for item in state["queue_items"]])++ strategy = BestFirstCrawlingStrategy(+ max_depth=2, max_pages=10, on_state_change=on_state_change+ )+ crawled = [+ r.url+ async for r in strategy._arun_stream(ROOT, make_crawler(link_map), make_config(stream=True))+ ]++ for queue in snapshots:+ assert not [u for u, n in Counter(queue).items() if n > 1], f"duplicate in queue: {queue}"+ assert sorted(crawled) == sorted(link_map)