Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): pre-warm robots.txt cache before crawl loop starts

Abdullaha86e9709ea04
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index cf1715e..163101b 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -97,7 +97,10 @@ class CrawlerEngine:
if domain in self._domain_delays:
return self._domain_delays[domain]
- # Fetch both robots.txt directives in a single parser lookup
+ # For domains covered by _prefetch_robots_txt this is a local parser read.
+ # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.
+ # Two concurrent callbacks hitting the same new domain can each trigger a fetch;
+ # the second write is a no-op in effect (same content), but the extra request is accepted.
c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid)
delay = self.spider.download_delay
@@ -287,6 +290,27 @@ class CrawlerEngine:
return True
+ async def _prefetch_robots_txt(self) -> None:
+ """Pre-warm the robots.txt cache before the crawl loop starts.
+
+ Uses allowed_domains if configured, otherwise falls back to unique domains
+ extracted from start_urls via Request.domain. Both paths use https.
+ """
+ if not self._robots_manager:
+ return
+
+ if self._allowed_domains:
+ domains = self._allowed_domains
+ elif self.spider.start_urls:
+ # Deduplicate by domain so we spawn exactly one task per domain
+ domains = {Request(url).domain for url in self.spider.start_urls}
+ else:
+ return
+
+ seed_urls = [f"https://{domain}/" for domain in domains]
+
+ await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
+
async def crawl(self) -> CrawlStats:
"""Run the spider and return CrawlStats."""
self._running = True
@@ -310,6 +334,8 @@ class CrawlerEngine:
await self.spider.on_start(resuming=resuming)
+ await self._prefetch_robots_txt()
+
try:
if not resuming:
async for request in self.spider.start_requests():
diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.py
index e362036..e7382f8 100644
--- a/tests/spiders/test_engine.py
+++ b/tests/spiders/test_engine.py
@@ -8,6 +8,7 @@ import pytest
from scrapling.spiders.engine import CrawlerEngine, _dump
from scrapling.spiders.request import Request
+from scrapling.spiders.robotstxt import RobotsTxtManager
from scrapling.spiders.session import SessionManager
from scrapling.spiders.result import CrawlStats, ItemList
from scrapling.spiders.checkpoint import CheckpointData
@@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGenerator
class MockResponse:
"""Minimal Response stand-in."""
- def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"):
+ def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"):
self.status = status
self.body = body
self.url = url
+ self.encoding = encoding
self.request: Any = None
self.meta: Dict[str, Any] = {}
@@ -84,6 +86,7 @@ class MockSpider:
on_scraped_item_fn=None,
retry_blocked_request_fn=None,
robots_txt_obey: bool = False,
+ start_urls: list[str] | None = None,
):
self.concurrent_requests = concurrent_requests
self.concurrent_requests_per_domain = concurrent_requests_per_domain
@@ -95,6 +98,7 @@ class MockSpider:
self.fp_keep_fragments = fp_keep_fragments
self.name = "test_spider"
self.robots_txt_obey = robots_txt_obey
+ self.start_urls = start_urls or []
# Tracking lists
self.on_start_calls: list[dict] = []
@@ -914,3 +918,83 @@ class TestPauseDuringCrawl:
await engine.crawl()
assert engine.paused is False
+
+
+# ---------------------------------------------------------------------------
+# Tests: _prefetch_robots_txt
+# ---------------------------------------------------------------------------
+
+
+class TestPrefetchRobotsTxt:
+ """_prefetch_robots_txt warms the robots.txt cache before the crawl loop."""
+
+ @staticmethod
+ def _make_counting_fetch():
+ """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair."""
+ calls: list[tuple[str, str]] = []
+
+ async def _fetch(url: str, sid: str):
+ calls.append((url, sid))
+ return MockResponse(status=200, body=b"", url=url)
+
+ return _fetch, calls
+
+ @pytest.mark.asyncio
+ async def test_prefetch_uses_allowed_domains_when_set(self):
+ fetch_fn, calls = self._make_counting_fetch()
+ spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)
+ engine = _make_engine(spider=spider)
+ engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+ await engine._prefetch_robots_txt()
+
+ fetched_domains = {Request(url).domain for url, _ in calls}
+ assert fetched_domains == {"a.com", "b.com"}
+
+ @pytest.mark.asyncio
+ async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):
+ fetch_fn, calls = self._make_counting_fetch()
+ spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])
+ engine = _make_engine(spider=spider)
+ engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+ await engine._prefetch_robots_txt()
+
+ assert len(calls) == 1
+ assert calls[0][0] == "https://example.com/robots.txt"
+
+ @pytest.mark.asyncio
+ async def test_prefetch_noop_when_robots_disabled(self):
+ fetch_fn, calls = self._make_counting_fetch()
+ spider = MockSpider(robots_txt_obey=False)
+ engine = _make_engine(spider=spider)
+
+ assert engine._robots_manager is None
+
+ await engine._prefetch_robots_txt()
+
+ assert calls == []
+
+ @pytest.mark.asyncio
+ async def test_prefetch_noop_when_start_urls_empty(self):
+ fetch_fn, calls = self._make_counting_fetch()
+ spider = MockSpider(robots_txt_obey=True, start_urls=[])
+ engine = _make_engine(spider=spider)
+ engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+ await engine._prefetch_robots_txt()
+
+ assert calls == []
+
+ @pytest.mark.asyncio
+ async def test_prefetch_deduplicates_same_domain_in_start_urls(self):
+ fetch_fn, calls = self._make_counting_fetch()
+ spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"])
+ engine = _make_engine(spider=spider)
+ engine._robots_manager = RobotsTxtManager(fetch_fn)
+
+ await engine._prefetch_robots_txt()
+
+ # set of Request.domain values deduplicates to one task per domain
+ assert len(calls) == 1
+ assert calls[0][0] == "https://example.com/robots.txt"