🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(CrawlSpider): Add crawl strategy with values BFS or DFS
Karim shoair7183891c31c8
diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zipindex 3cd3281..252d327 100644Binary files a/agent-skill/Scrapling-Skill.zip and b/agent-skill/Scrapling-Skill.zip differdiff --git a/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.mdindex 5823c3e..df465ef 100644--- a/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md+++ b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md@@ -36,6 +36,21 @@ A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound methoIf a rule has no callback, the matched URLs fall through to the spider's default `parse()` (or stay uncallback'd if you didn't override it). This is convenient for pagination: extract the next-page links to keep the crawl going, but don't need a separate handler.+### Crawl order++Set `crawl_strategy` on `CrawlSpider` or its subclasses, including `ContentSpider`, to choose the next pending request:++| `crawl_strategy` | Order within the same `Request.priority` |+|------------------|------------------------------------------|+| `"bfs"` (default) | Oldest queued request first |+| `"dfs"` | Newest queued request first |++The scheduler takes pending requests with higher `Request.priority` first. The strategy only orders requests with equal priority.++Set `concurrent_requests = 1` to process one request at a time in the selected queue order. Duplicate URLs are still filtered; finding a URL again does not move its queued request. With parallel requests, pages can finish in a different order, and levels or branches can overlap.++Keep the same `crawl_strategy` when resuming from a checkpoint.+### Combining rules with custom logicOverride `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields:diff --git a/docs/spiders/generic-templates.md b/docs/spiders/generic-templates.mdindex 454e104..a05aa36 100644--- a/docs/spiders/generic-templates.md+++ b/docs/spiders/generic-templates.md@@ -37,6 +37,21 @@ A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound methoIf a rule has no callback, the matched URLs fall through to the spider's default `parse()`. This is convenient for pagination: extract the next-page links to keep the crawl going, without needing a separate handler.+### Crawl order++Set `crawl_strategy` on `CrawlSpider` or its subclasses, including `ContentSpider`, to choose the next pending request:++| `crawl_strategy` | Order within the same `Request.priority` |+|------------------|------------------------------------------|+| `"bfs"` (default) | Oldest queued request first |+| `"dfs"` | Newest queued request first |++The scheduler takes pending requests with higher `Request.priority` first. The strategy only orders requests with equal priority.++Set `concurrent_requests = 1` to process one request at a time in the selected queue order. Duplicate URLs are still filtered; finding a URL again does not move its queued request. With parallel requests, pages can finish in a different order, and levels or branches can overlap.++Keep the same `crawl_strategy` when resuming from a checkpoint.+### Combining rules with custom logicOverride `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields:diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 1b38b39..73eb390 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -42,6 +42,7 @@ class CrawlerEngine:include_kwargs=spider.fp_include_kwargs,include_headers=spider.fp_include_headers,keep_fragments=spider.fp_keep_fragments,+ crawl_strategy=getattr(spider, "crawl_strategy", "bfs"),)self.stats = CrawlStats()diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.pyindex 0484a68..ae72dde 100644--- a/scrapling/spiders/scheduler.py+++ b/scrapling/spiders/scheduler.py@@ -3,7 +3,7 @@ from itertools import countfrom scrapling.core.utils import logfrom scrapling.spiders.request import Request-from scrapling.core._types import List, Set, Tuple, TYPE_CHECKING+from scrapling.core._types import List, Set, Tuple, Literal, TYPE_CHECKINGif TYPE_CHECKING:from scrapling.spiders.checkpoint import CheckpointData@@ -14,13 +14,23 @@ class Scheduler:Priority queue with URL deduplication. (heapq)Higher priority requests are processed first.+ For equal priorities, bfs takes the oldest queued request and dfs takes the newest.Duplicate URLs are filtered unless dont_filter=True."""- def __init__(self, include_kwargs: bool = False, include_headers: bool = False, keep_fragments: bool = False):+ def __init__(+ self,+ include_kwargs: bool = False,+ include_headers: bool = False,+ keep_fragments: bool = False,+ crawl_strategy: Literal["bfs", "dfs"] = "bfs",+ ):+ if crawl_strategy not in ("bfs", "dfs"):+ raise ValueError("crawl_strategy must be 'bfs' or 'dfs'")self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()self._seen: set[bytes] = set()- self._counter = count()+ self._lifo = crawl_strategy == "dfs"+ self._counter = count(step=-1 if self._lifo else 1)# Mirror dict for snapshot without draining queueself._pending: dict[int, tuple[int, int, Request]] = {}self._inflight: dict[int, list[int]] = {}@@ -81,8 +91,7 @@ class Scheduler:"""self._seen = data.seen.copy()- # Restore pending requests in order (they're already sorted by priority)- for request in data.requests:+ for request in reversed(data.requests) if self._lifo else data.requests:counter = next(self._counter)item = (-request.priority, counter, request)self._pending[counter] = itemdiff --git a/scrapling/spiders/templates/crawler.py b/scrapling/spiders/templates/crawler.pyindex 5c6aeaa..b3c09b5 100644--- a/scrapling/spiders/templates/crawler.py+++ b/scrapling/spiders/templates/crawler.py@@ -12,6 +12,7 @@ from scrapling.core._types import (Callable,Dict,List,+ Literal,Optional,Union,)@@ -57,6 +58,8 @@ class CrawlSpider(Spider):You can start from it and override it as needed for more custom functionality, or just implement your own spider."""+ crawl_strategy: Literal["bfs", "dfs"] = "bfs"+def rules(self) -> List[CrawlRule]:"""Override to define link-following rules."""return []diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.pyindex 8e4bac2..ef8e152 100644--- a/tests/spiders/test_engine.py+++ b/tests/spiders/test_engine.py@@ -10,6 +10,7 @@ from scrapling.spiders.engine import CrawlerEngine, _dumpfrom scrapling.spiders.request import Requestfrom scrapling.spiders.robotstxt import RobotsTxtManagerfrom scrapling.spiders.session import SessionManager+from scrapling.spiders.scheduler import Schedulerfrom scrapling.spiders.result import CrawlStats, ItemListfrom scrapling.spiders.checkpoint import CheckpointDatafrom scrapling.core._types import Any, Dict, Set, AsyncGenerator@@ -378,16 +379,27 @@ class TestProcessRequest:assert engine.stats.requests_count == 0@pytest.mark.asyncio- async def test_blocked_response_triggers_retry(self):+ @pytest.mark.parametrize("strategy", ["bfs", "dfs"])+ async def test_blocked_response_triggers_retry(self, strategy):spider = MockSpider(is_blocked_fn=lambda r: True, max_blocked_retries=2)engine = _make_engine(spider=spider)+ engine.scheduler = Scheduler(crawl_strategy=strategy)+ other = Request("https://example.com/other", sid="default")+ await engine.scheduler.enqueue(other)request = Request("https://example.com", sid="default")await engine._process_request(request)assert engine.stats.blocked_requests_count == 1# A retry request should be enqueuedassert not engine.scheduler.is_empty+ assert await engine.scheduler.dequeue() is other+ retry = await engine.scheduler.dequeue()+ assert retry.url == request.url+ assert retry.priority == -1+ assert request.priority == 0+ assert retry._retry_count == 1+ assert retry.dont_filter@pytest.mark.asyncioasync def test_blocked_response_max_retries_exceeded(self):diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.pyindex 3646f59..ffb8504 100644--- a/tests/spiders/test_scheduler.py+++ b/tests/spiders/test_scheduler.py@@ -1,5 +1,7 @@"""Tests for the Scheduler class."""+import pickle+import pytestfrom scrapling.spiders.request import Request@@ -17,6 +19,61 @@ class TestSchedulerInit:assert len(scheduler) == 0assert scheduler.is_empty is True+ @pytest.mark.parametrize("strategy", ["", "BFS", "random", None])+ def test_invalid_crawl_strategy(self, strategy):+ with pytest.raises(ValueError, match="crawl_strategy must be 'bfs' or 'dfs'"):+ Scheduler(crawl_strategy=strategy)+++class TestSchedulerOrder:+ @pytest.mark.asyncio+ @pytest.mark.parametrize("strategy, order", [("bfs", [0, 1, 4, 3]), ("dfs", [4, 1, 0, 3])])+ async def test_arrival_order_preserves_priorities(self, strategy, order):+ scheduler = Scheduler(crawl_strategy=strategy)+ requests = [Request(f"https://example.com/{i}", priority=p) for i, p in enumerate([5, 5, 10, 1, 5])]+ for request in requests[:4]:+ await scheduler.enqueue(request)+ first = await scheduler.dequeue()+ assert first is requests[2]+ scheduler.complete(first)+ await scheduler.enqueue(requests[4])++ for i in order:+ request = await scheduler.dequeue()+ assert request is requests[i]+ scheduler.complete(request)++ assert [request.priority for request in requests] == [5, 5, 10, 1, 5]+ assert scheduler.is_empty+ assert scheduler.snapshot()[0] == []++ @pytest.mark.asyncio+ @pytest.mark.parametrize("strategy, order", [("bfs", [1, 2, 4, 0, 3]), ("dfs", [4, 2, 1, 3, 0])])+ async def test_resume_keeps_order_and_unfinished_requests(self, strategy, order):+ scheduler = Scheduler(crawl_strategy=strategy)+ requests = [Request(f"https://example.com/{i}", priority=p) for i, p in enumerate([1, 5, 5, 1, 5])]+ for request in requests[:4]:+ await scheduler.enqueue(request)+ unfinished = await scheduler.dequeue()+ pending, seen = scheduler.snapshot()+ assert unfinished in pending+ data = pickle.loads(pickle.dumps(CheckpointData(requests=pending, seen=seen)))+ saved_urls = [request.url for request in data.requests]+ restored = Scheduler(crawl_strategy=strategy)+ restored.restore(data)+ assert [request.url for request in data.requests] == saved_urls+ assert [request.url for request in restored.snapshot()[0]] == saved_urls+ assert not await restored.enqueue(Request(requests[0].url))+ await restored.enqueue(requests[4])++ for i in order:+ request = await restored.dequeue()+ assert request.url == requests[i].url+ restored.complete(request)++ assert restored.is_empty+ assert restored.snapshot()[0] == []+class TestSchedulerEnqueue:"""Test Scheduler enqueue functionality."""@@ -425,9 +482,10 @@ class TestSchedulerInFlight:assert requests == []@pytest.mark.asyncio- async def test_duplicate_instance_tracked_per_enqueue(self):+ @pytest.mark.parametrize("strategy", ["bfs", "dfs"])+ async def test_duplicate_instance_tracked_per_enqueue(self, strategy):"""Re-enqueuing the same Request instance (dont_filter) tracks each copy."""- scheduler = Scheduler()+ scheduler = Scheduler(crawl_strategy=strategy)request = Request("https://example.com/retry", dont_filter=True)await scheduler.enqueue(request)@@ -441,6 +499,9 @@ class TestSchedulerInFlight:requests, _ = scheduler.snapshot()assert len(requests) == 1+ scheduler.complete(await scheduler.dequeue())+ assert scheduler.snapshot()[0] == []+@pytest.mark.asyncioasync def test_in_flight_request_survives_checkpoint_resume(self):"""Regression (F1): a request in-flight at checkpoint time survives resume."""diff --git a/tests/spiders/test_site_to_markdown.py b/tests/spiders/test_site_to_markdown.pyindex aa0f082..69f8e80 100644--- a/tests/spiders/test_site_to_markdown.py+++ b/tests/spiders/test_site_to_markdown.py@@ -4,6 +4,7 @@ import pytestfrom scrapling.engines.toolbelt.custom import Responsefrom scrapling.spiders.links import LinkExtractor+from scrapling.spiders.engine import CrawlerEnginefrom scrapling.spiders.request import Requestfrom scrapling.spiders.templates import CrawlRule, ContentSpiderfrom scrapling.core._types import AsyncGenerator@@ -51,6 +52,21 @@ def _spider(**attrs) -> ContentSpider:class TestContentSpider:+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "strategy, paths", [("bfs", ["/docs/page-1", "/blog/post-1"]), ("dfs", ["/blog/post-1", "/docs/page-1"])]+ )+ async def test_inherited_crawl_strategy(self, strategy, paths):+ spider = _spider(crawl_strategy=strategy)+ engine = CrawlerEngine(spider, spider._session_manager)+ response = _make_response()+ await engine._run_callbacks(response.request, response)++ assert engine.stats.items_scraped == 1+ assert [(await engine.scheduler.dequeue()).url for _ in paths] == [+ f"https://example.com{path}" for path in paths+ ]+def test_requires_allowed_domains(self):class S(ContentSpider):name = "s"diff --git a/tests/spiders/test_templates.py b/tests/spiders/test_templates.pyindex 95f3e21..008b178 100644--- a/tests/spiders/test_templates.py+++ b/tests/spiders/test_templates.py@@ -1,6 +1,7 @@"""Tests for `CrawlSpider` and `CrawlRule`."""import pickle+from unittest.mock import AsyncMockimport pytest@@ -21,11 +22,11 @@ HTML = """"""-def _make_response(url: str = "https://example.com/") -> Response:+def _make_response(url: str = "https://example.com/", html: str = HTML) -> Response:"""Build a Response with a Request attached so `response.follow()` works."""resp = Response(url=url,- content=HTML,+ content=html,status=200,reason="OK",cookies={},@@ -52,6 +53,49 @@ async def _collect(agen: AsyncGenerator) -> list:class TestCrawlSpider:+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "strategy, order", [("bfs", ["/", "/a", "/b", "/a/1", "/b/1"]), ("dfs", ["/", "/b", "/b/1", "/a", "/a/1"])]+ )+ async def test_crawl_strategy_with_callbacks(self, strategy, order):+ pages = {+ "/": '<a href="/a">A</a><a href="/b">B</a>',+ "/a": '<a href="/a/1">A1</a>',+ "/b": '<a href="/b/1">B1</a>',+ "/a/1": '<a href="/">Home</a>',+ "/b/1": '<a href="/">Home</a>',+ }++ async def fetch(url, **kwargs):+ return _make_response(url, pages[url.removeprefix("https://example.com")])++ session = AsyncMock()+ session._is_alive = False+ session.fetch.side_effect = fetch++ class OrderedSpider(CrawlSpider):+ name = "ordered"+ start_urls = ["https://example.com/"]+ concurrent_requests = 1+ crawl_strategy = strategy++ def configure_sessions(self, manager):+ manager.add("graph", session)++ def rules(self):+ return [CrawlRule(LinkExtractor(), callback=self.parse_page)]++ async def parse_page(self, response):+ yield {"url": response.url}+ async for request in self.parse(response):+ yield request++ items = [item async for item in OrderedSpider().stream()]+ urls = [f"https://example.com{path}" for path in order]+ assert [call.kwargs["url"] for call in session.fetch.await_args_list] == urls+ assert [item["url"] for item in items] == urls[1:]+ session.__aexit__.assert_awaited_once()+@pytest.mark.asyncioasync def test_empty_rules_yields_nothing(self):class S(CrawlSpider):