Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(CrawlSpider): Add crawl strategy with values BFS or DFS

Karim shoair7183891c31c8
diff --git a/agent-skill/Scrapling-Skill.zip b/agent-skill/Scrapling-Skill.zip
index 3cd3281..252d327 100644
Binary files a/agent-skill/Scrapling-Skill.zip and b/agent-skill/Scrapling-Skill.zip differ
diff --git a/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md
index 5823c3e..df465ef 100644
--- a/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md
+++ b/agent-skill/Scrapling-Skill/references/spiders/generic-templates.md
@@ -36,6 +36,21 @@ A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound metho
If a rule has no callback, the matched URLs fall through to the spider's default `parse()` (or stay uncallback'd if you didn't override it). This is convenient for pagination: extract the next-page links to keep the crawl going, but don't need a separate handler.
+### Crawl order
+
+Set `crawl_strategy` on `CrawlSpider` or its subclasses, including `ContentSpider`, to choose the next pending request:
+
+| `crawl_strategy` | Order within the same `Request.priority` |
+|------------------|------------------------------------------|
+| `"bfs"` (default) | Oldest queued request first |
+| `"dfs"` | Newest queued request first |
+
+The scheduler takes pending requests with higher `Request.priority` first. The strategy only orders requests with equal priority.
+
+Set `concurrent_requests = 1` to process one request at a time in the selected queue order. Duplicate URLs are still filtered; finding a URL again does not move its queued request. With parallel requests, pages can finish in a different order, and levels or branches can overlap.
+
+Keep the same `crawl_strategy` when resuming from a checkpoint.
+
### Combining rules with custom logic
Override `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields:
diff --git a/docs/spiders/generic-templates.md b/docs/spiders/generic-templates.md
index 454e104..a05aa36 100644
--- a/docs/spiders/generic-templates.md
+++ b/docs/spiders/generic-templates.md
@@ -37,6 +37,21 @@ A `CrawlRule` pairs a `LinkExtractor` with an optional `callback` (a bound metho
If a rule has no callback, the matched URLs fall through to the spider's default `parse()`. This is convenient for pagination: extract the next-page links to keep the crawl going, without needing a separate handler.
+### Crawl order
+
+Set `crawl_strategy` on `CrawlSpider` or its subclasses, including `ContentSpider`, to choose the next pending request:
+
+| `crawl_strategy` | Order within the same `Request.priority` |
+|------------------|------------------------------------------|
+| `"bfs"` (default) | Oldest queued request first |
+| `"dfs"` | Newest queued request first |
+
+The scheduler takes pending requests with higher `Request.priority` first. The strategy only orders requests with equal priority.
+
+Set `concurrent_requests = 1` to process one request at a time in the selected queue order. Duplicate URLs are still filtered; finding a URL again does not move its queued request. With parallel requests, pages can finish in a different order, and levels or branches can overlap.
+
+Keep the same `crawl_strategy` when resuming from a checkpoint.
+
### Combining rules with custom logic
Override `parse()` and call `super().parse(response)` to get the rule behavior plus your own yields:
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 1b38b39..73eb390 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -42,6 +42,7 @@ class CrawlerEngine:
include_kwargs=spider.fp_include_kwargs,
include_headers=spider.fp_include_headers,
keep_fragments=spider.fp_keep_fragments,
+ crawl_strategy=getattr(spider, "crawl_strategy", "bfs"),
)
self.stats = CrawlStats()
diff --git a/scrapling/spiders/scheduler.py b/scrapling/spiders/scheduler.py
index 0484a68..ae72dde 100644
--- a/scrapling/spiders/scheduler.py
+++ b/scrapling/spiders/scheduler.py
@@ -3,7 +3,7 @@ from itertools import count
from scrapling.core.utils import log
from scrapling.spiders.request import Request
-from scrapling.core._types import List, Set, Tuple, TYPE_CHECKING
+from scrapling.core._types import List, Set, Tuple, Literal, TYPE_CHECKING
if TYPE_CHECKING:
from scrapling.spiders.checkpoint import CheckpointData
@@ -14,13 +14,23 @@ class Scheduler:
Priority queue with URL deduplication. (heapq)
Higher priority requests are processed first.
+ For equal priorities, bfs takes the oldest queued request and dfs takes the newest.
Duplicate URLs are filtered unless dont_filter=True.
"""
- def __init__(self, include_kwargs: bool = False, include_headers: bool = False, keep_fragments: bool = False):
+ def __init__(
+ self,
+ include_kwargs: bool = False,
+ include_headers: bool = False,
+ keep_fragments: bool = False,
+ crawl_strategy: Literal["bfs", "dfs"] = "bfs",
+ ):
+ if crawl_strategy not in ("bfs", "dfs"):
+ raise ValueError("crawl_strategy must be 'bfs' or 'dfs'")
self._queue: asyncio.PriorityQueue[tuple[int, int, Request]] = asyncio.PriorityQueue()
self._seen: set[bytes] = set()
- self._counter = count()
+ self._lifo = crawl_strategy == "dfs"
+ self._counter = count(step=-1 if self._lifo else 1)
# Mirror dict for snapshot without draining queue
self._pending: dict[int, tuple[int, int, Request]] = {}
self._inflight: dict[int, list[int]] = {}
@@ -81,8 +91,7 @@ class Scheduler:
"""
self._seen = data.seen.copy()
- # Restore pending requests in order (they're already sorted by priority)
- for request in data.requests:
+ for request in reversed(data.requests) if self._lifo else data.requests:
counter = next(self._counter)
item = (-request.priority, counter, request)
self._pending[counter] = item
diff --git a/scrapling/spiders/templates/crawler.py b/scrapling/spiders/templates/crawler.py
index 5c6aeaa..b3c09b5 100644
--- a/scrapling/spiders/templates/crawler.py
+++ b/scrapling/spiders/templates/crawler.py
@@ -12,6 +12,7 @@ from scrapling.core._types import (
Callable,
Dict,
List,
+ Literal,
Optional,
Union,
)
@@ -57,6 +58,8 @@ class CrawlSpider(Spider):
You can start from it and override it as needed for more custom functionality, or just implement your own spider.
"""
+ crawl_strategy: Literal["bfs", "dfs"] = "bfs"
+
def rules(self) -> List[CrawlRule]:
"""Override to define link-following rules."""
return []
diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py
index 8e4bac2..ef8e152 100644
--- a/tests/spiders/test_engine.py
+++ b/tests/spiders/test_engine.py
@@ -10,6 +10,7 @@ from scrapling.spiders.engine import CrawlerEngine, _dump
from scrapling.spiders.request import Request
from scrapling.spiders.robotstxt import RobotsTxtManager
from scrapling.spiders.session import SessionManager
+from scrapling.spiders.scheduler import Scheduler
from scrapling.spiders.result import CrawlStats, ItemList
from scrapling.spiders.checkpoint import CheckpointData
from scrapling.core._types import Any, Dict, Set, AsyncGenerator
@@ -378,16 +379,27 @@ class TestProcessRequest:
assert engine.stats.requests_count == 0
@pytest.mark.asyncio
- async def test_blocked_response_triggers_retry(self):
+ @pytest.mark.parametrize("strategy", ["bfs", "dfs"])
+ async def test_blocked_response_triggers_retry(self, strategy):
spider = MockSpider(is_blocked_fn=lambda r: True, max_blocked_retries=2)
engine = _make_engine(spider=spider)
+ engine.scheduler = Scheduler(crawl_strategy=strategy)
+ other = Request("https://example.com/other", sid="default")
+ await engine.scheduler.enqueue(other)
request = Request("https://example.com", sid="default")
await engine._process_request(request)
assert engine.stats.blocked_requests_count == 1
# A retry request should be enqueued
assert not engine.scheduler.is_empty
+ assert await engine.scheduler.dequeue() is other
+ retry = await engine.scheduler.dequeue()
+ assert retry.url == request.url
+ assert retry.priority == -1
+ assert request.priority == 0
+ assert retry._retry_count == 1
+ assert retry.dont_filter
@pytest.mark.asyncio
async def test_blocked_response_max_retries_exceeded(self):
diff --git a/tests/spiders/test_scheduler.py b/tests/spiders/test_scheduler.py
index 3646f59..ffb8504 100644
--- a/tests/spiders/test_scheduler.py
+++ b/tests/spiders/test_scheduler.py
@@ -1,5 +1,7 @@
"""Tests for the Scheduler class."""
+import pickle
+
import pytest
from scrapling.spiders.request import Request
@@ -17,6 +19,61 @@ class TestSchedulerInit:
assert len(scheduler) == 0
assert scheduler.is_empty is True
+ @pytest.mark.parametrize("strategy", ["", "BFS", "random", None])
+ def test_invalid_crawl_strategy(self, strategy):
+ with pytest.raises(ValueError, match="crawl_strategy must be 'bfs' or 'dfs'"):
+ Scheduler(crawl_strategy=strategy)
+
+
+class TestSchedulerOrder:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("strategy, order", [("bfs", [0, 1, 4, 3]), ("dfs", [4, 1, 0, 3])])
+ async def test_arrival_order_preserves_priorities(self, strategy, order):
+ scheduler = Scheduler(crawl_strategy=strategy)
+ requests = [Request(f"https://example.com/{i}", priority=p) for i, p in enumerate([5, 5, 10, 1, 5])]
+ for request in requests[:4]:
+ await scheduler.enqueue(request)
+ first = await scheduler.dequeue()
+ assert first is requests[2]
+ scheduler.complete(first)
+ await scheduler.enqueue(requests[4])
+
+ for i in order:
+ request = await scheduler.dequeue()
+ assert request is requests[i]
+ scheduler.complete(request)
+
+ assert [request.priority for request in requests] == [5, 5, 10, 1, 5]
+ assert scheduler.is_empty
+ assert scheduler.snapshot()[0] == []
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize("strategy, order", [("bfs", [1, 2, 4, 0, 3]), ("dfs", [4, 2, 1, 3, 0])])
+ async def test_resume_keeps_order_and_unfinished_requests(self, strategy, order):
+ scheduler = Scheduler(crawl_strategy=strategy)
+ requests = [Request(f"https://example.com/{i}", priority=p) for i, p in enumerate([1, 5, 5, 1, 5])]
+ for request in requests[:4]:
+ await scheduler.enqueue(request)
+ unfinished = await scheduler.dequeue()
+ pending, seen = scheduler.snapshot()
+ assert unfinished in pending
+ data = pickle.loads(pickle.dumps(CheckpointData(requests=pending, seen=seen)))
+ saved_urls = [request.url for request in data.requests]
+ restored = Scheduler(crawl_strategy=strategy)
+ restored.restore(data)
+ assert [request.url for request in data.requests] == saved_urls
+ assert [request.url for request in restored.snapshot()[0]] == saved_urls
+ assert not await restored.enqueue(Request(requests[0].url))
+ await restored.enqueue(requests[4])
+
+ for i in order:
+ request = await restored.dequeue()
+ assert request.url == requests[i].url
+ restored.complete(request)
+
+ assert restored.is_empty
+ assert restored.snapshot()[0] == []
+
class TestSchedulerEnqueue:
"""Test Scheduler enqueue functionality."""
@@ -425,9 +482,10 @@ class TestSchedulerInFlight:
assert requests == []
@pytest.mark.asyncio
- async def test_duplicate_instance_tracked_per_enqueue(self):
+ @pytest.mark.parametrize("strategy", ["bfs", "dfs"])
+ async def test_duplicate_instance_tracked_per_enqueue(self, strategy):
"""Re-enqueuing the same Request instance (dont_filter) tracks each copy."""
- scheduler = Scheduler()
+ scheduler = Scheduler(crawl_strategy=strategy)
request = Request("https://example.com/retry", dont_filter=True)
await scheduler.enqueue(request)
@@ -441,6 +499,9 @@ class TestSchedulerInFlight:
requests, _ = scheduler.snapshot()
assert len(requests) == 1
+ scheduler.complete(await scheduler.dequeue())
+ assert scheduler.snapshot()[0] == []
+
@pytest.mark.asyncio
async def test_in_flight_request_survives_checkpoint_resume(self):
"""Regression (F1): a request in-flight at checkpoint time survives resume."""
diff --git a/tests/spiders/test_site_to_markdown.py b/tests/spiders/test_site_to_markdown.py
index aa0f082..69f8e80 100644
--- a/tests/spiders/test_site_to_markdown.py
+++ b/tests/spiders/test_site_to_markdown.py
@@ -4,6 +4,7 @@ import pytest
from scrapling.engines.toolbelt.custom import Response
from scrapling.spiders.links import LinkExtractor
+from scrapling.spiders.engine import CrawlerEngine
from scrapling.spiders.request import Request
from scrapling.spiders.templates import CrawlRule, ContentSpider
from scrapling.core._types import AsyncGenerator
@@ -51,6 +52,21 @@ def _spider(**attrs) -> ContentSpider:
class TestContentSpider:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "strategy, paths", [("bfs", ["/docs/page-1", "/blog/post-1"]), ("dfs", ["/blog/post-1", "/docs/page-1"])]
+ )
+ async def test_inherited_crawl_strategy(self, strategy, paths):
+ spider = _spider(crawl_strategy=strategy)
+ engine = CrawlerEngine(spider, spider._session_manager)
+ response = _make_response()
+ await engine._run_callbacks(response.request, response)
+
+ assert engine.stats.items_scraped == 1
+ assert [(await engine.scheduler.dequeue()).url for _ in paths] == [
+ f"https://example.com{path}" for path in paths
+ ]
+
def test_requires_allowed_domains(self):
class S(ContentSpider):
name = "s"
diff --git a/tests/spiders/test_templates.py b/tests/spiders/test_templates.py
index 95f3e21..008b178 100644
--- a/tests/spiders/test_templates.py
+++ b/tests/spiders/test_templates.py
@@ -1,6 +1,7 @@
"""Tests for `CrawlSpider` and `CrawlRule`."""
import pickle
+from unittest.mock import AsyncMock
import pytest
@@ -21,11 +22,11 @@ HTML = """
"""
-def _make_response(url: str = "https://example.com/") -> Response:
+def _make_response(url: str = "https://example.com/", html: str = HTML) -> Response:
"""Build a Response with a Request attached so `response.follow()` works."""
resp = Response(
url=url,
- content=HTML,
+ content=html,
status=200,
reason="OK",
cookies={},
@@ -52,6 +53,49 @@ async def _collect(agen: AsyncGenerator) -> list:
class TestCrawlSpider:
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "strategy, order", [("bfs", ["/", "/a", "/b", "/a/1", "/b/1"]), ("dfs", ["/", "/b", "/b/1", "/a", "/a/1"])]
+ )
+ async def test_crawl_strategy_with_callbacks(self, strategy, order):
+ pages = {
+ "/": '<a href="/a">A</a><a href="/b">B</a>',
+ "/a": '<a href="/a/1">A1</a>',
+ "/b": '<a href="/b/1">B1</a>',
+ "/a/1": '<a href="/">Home</a>',
+ "/b/1": '<a href="/">Home</a>',
+ }
+
+ async def fetch(url, **kwargs):
+ return _make_response(url, pages[url.removeprefix("https://example.com")])
+
+ session = AsyncMock()
+ session._is_alive = False
+ session.fetch.side_effect = fetch
+
+ class OrderedSpider(CrawlSpider):
+ name = "ordered"
+ start_urls = ["https://example.com/"]
+ concurrent_requests = 1
+ crawl_strategy = strategy
+
+ def configure_sessions(self, manager):
+ manager.add("graph", session)
+
+ def rules(self):
+ return [CrawlRule(LinkExtractor(), callback=self.parse_page)]
+
+ async def parse_page(self, response):
+ yield {"url": response.url}
+ async for request in self.parse(response):
+ yield request
+
+ items = [item async for item in OrderedSpider().stream()]
+ urls = [f"https://example.com{path}" for path in order]
+ assert [call.kwargs["url"] for call in session.fetch.await_args_list] == urls
+ assert [item["url"] for item in items] == urls[1:]
+ session.__aexit__.assert_awaited_once()
+
@pytest.mark.asyncio
async def test_empty_rules_yields_nothing(self):
class S(CrawlSpider):