🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): pre-warm robots.txt cache before crawl loop starts
Abdullaha86e9709ea04
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex cf1715e..163101b 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -97,7 +97,10 @@ class CrawlerEngine:if domain in self._domain_delays:return self._domain_delays[domain]- # Fetch both robots.txt directives in a single parser lookup+ # For domains covered by _prefetch_robots_txt this is a local parser read.+ # Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.+ # Two concurrent callbacks hitting the same new domain can each trigger a fetch;+ # the second write is a no-op in effect (same content), but the extra request is accepted.c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid)delay = self.spider.download_delay@@ -287,6 +290,27 @@ class CrawlerEngine:return True+ async def _prefetch_robots_txt(self) -> None:+ """Pre-warm the robots.txt cache before the crawl loop starts.++ Uses allowed_domains if configured, otherwise falls back to unique domains+ extracted from start_urls via Request.domain. Both paths use https.+ """+ if not self._robots_manager:+ return++ if self._allowed_domains:+ domains = self._allowed_domains+ elif self.spider.start_urls:+ # Deduplicate by domain so we spawn exactly one task per domain+ domains = {Request(url).domain for url in self.spider.start_urls}+ else:+ return++ seed_urls = [f"https://{domain}/" for domain in domains]++ await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)+async def crawl(self) -> CrawlStats:"""Run the spider and return CrawlStats."""self._running = True@@ -310,6 +334,8 @@ class CrawlerEngine:await self.spider.on_start(resuming=resuming)+ await self._prefetch_robots_txt()+try:if not resuming:async for request in self.spider.start_requests():diff --git a/tests/spiders/test_engine.py b/tests/spiders/test_engine.pyindex e362036..e7382f8 100644--- a/tests/spiders/test_engine.py+++ b/tests/spiders/test_engine.py@@ -8,6 +8,7 @@ import pytestfrom scrapling.spiders.engine import CrawlerEngine, _dumpfrom scrapling.spiders.request import Request+from scrapling.spiders.robotstxt import RobotsTxtManagerfrom scrapling.spiders.session import SessionManagerfrom scrapling.spiders.result import CrawlStats, ItemListfrom scrapling.spiders.checkpoint import CheckpointData@@ -22,10 +23,11 @@ from scrapling.core._types import Any, Dict, Set, AsyncGeneratorclass MockResponse:"""Minimal Response stand-in."""- def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com"):+ def __init__(self, status: int = 200, body: bytes = b"ok", url: str = "https://example.com", encoding: str = "utf-8"):self.status = statusself.body = bodyself.url = url+ self.encoding = encodingself.request: Any = Noneself.meta: Dict[str, Any] = {}@@ -84,6 +86,7 @@ class MockSpider:on_scraped_item_fn=None,retry_blocked_request_fn=None,robots_txt_obey: bool = False,+ start_urls: list[str] | None = None,):self.concurrent_requests = concurrent_requestsself.concurrent_requests_per_domain = concurrent_requests_per_domain@@ -95,6 +98,7 @@ class MockSpider:self.fp_keep_fragments = fp_keep_fragmentsself.name = "test_spider"self.robots_txt_obey = robots_txt_obey+ self.start_urls = start_urls or []# Tracking listsself.on_start_calls: list[dict] = []@@ -914,3 +918,83 @@ class TestPauseDuringCrawl:await engine.crawl()assert engine.paused is False+++# ---------------------------------------------------------------------------+# Tests: _prefetch_robots_txt+# ---------------------------------------------------------------------------+++class TestPrefetchRobotsTxt:+ """_prefetch_robots_txt warms the robots.txt cache before the crawl loop."""++ @staticmethod+ def _make_counting_fetch():+ """Return (fetch_fn, calls_list) where calls_list records every (url, sid) pair."""+ calls: list[tuple[str, str]] = []++ async def _fetch(url: str, sid: str):+ calls.append((url, sid))+ return MockResponse(status=200, body=b"", url=url)++ return _fetch, calls++ @pytest.mark.asyncio+ async def test_prefetch_uses_allowed_domains_when_set(self):+ fetch_fn, calls = self._make_counting_fetch()+ spider = MockSpider(allowed_domains={"a.com", "b.com"}, robots_txt_obey=True)+ engine = _make_engine(spider=spider)+ engine._robots_manager = RobotsTxtManager(fetch_fn)++ await engine._prefetch_robots_txt()++ fetched_domains = {Request(url).domain for url, _ in calls}+ assert fetched_domains == {"a.com", "b.com"}++ @pytest.mark.asyncio+ async def test_prefetch_falls_back_to_start_urls_when_no_allowed_domains(self):+ fetch_fn, calls = self._make_counting_fetch()+ spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/page1"])+ engine = _make_engine(spider=spider)+ engine._robots_manager = RobotsTxtManager(fetch_fn)++ await engine._prefetch_robots_txt()++ assert len(calls) == 1+ assert calls[0][0] == "https://example.com/robots.txt"++ @pytest.mark.asyncio+ async def test_prefetch_noop_when_robots_disabled(self):+ fetch_fn, calls = self._make_counting_fetch()+ spider = MockSpider(robots_txt_obey=False)+ engine = _make_engine(spider=spider)++ assert engine._robots_manager is None++ await engine._prefetch_robots_txt()++ assert calls == []++ @pytest.mark.asyncio+ async def test_prefetch_noop_when_start_urls_empty(self):+ fetch_fn, calls = self._make_counting_fetch()+ spider = MockSpider(robots_txt_obey=True, start_urls=[])+ engine = _make_engine(spider=spider)+ engine._robots_manager = RobotsTxtManager(fetch_fn)++ await engine._prefetch_robots_txt()++ assert calls == []++ @pytest.mark.asyncio+ async def test_prefetch_deduplicates_same_domain_in_start_urls(self):+ fetch_fn, calls = self._make_counting_fetch()+ spider = MockSpider(robots_txt_obey=True, start_urls=["https://example.com/a", "https://example.com/b"])+ engine = _make_engine(spider=spider)+ engine._robots_manager = RobotsTxtManager(fetch_fn)++ await engine._prefetch_robots_txt()++ # set of Request.domain values deduplicates to one task per domain+ assert len(calls) == 1+ assert calls[0][0] == "https://example.com/robots.txt"