🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
feat(spiders): integrate robots.txt compliance into the crawl engine
Abdullah5c40c6a85392
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex d77f838..cf1715e 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -10,6 +10,7 @@ from scrapling.core.utils import logfrom scrapling.spiders.request import Requestfrom scrapling.spiders.scheduler import Schedulerfrom scrapling.spiders.session import SessionManager+from scrapling.spiders.robotstxt import RobotsTxtManagerfrom scrapling.spiders.result import CrawlStats, ItemListfrom scrapling.spiders.checkpoint import CheckpointManager, CheckpointDatafrom scrapling.core._types import Dict, Union, Optional, TYPE_CHECKING, Any, AsyncGenerator@@ -41,8 +42,18 @@ class CrawlerEngine:)self.stats = CrawlStats()+ if self.spider.robots_txt_obey:++ async def _fetch_robots(url: str, sid: str):+ return await self.session_manager.fetch(Request(url, sid=sid))++ self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots)+ else:+ self._robots_manager = None+self._global_limiter = CapacityLimiter(spider.concurrent_requests)self._domain_limiters: dict[str, CapacityLimiter] = {}+ self._domain_delays: dict[str, float] = {}self._allowed_domains: set[str] = spider.allowed_domains or set()self._active_tasks: int = 0@@ -68,13 +79,58 @@ class CrawlerEngine:return Truereturn False+ async def _get_domain_delay(self, request: Request) -> float:+ """Resolve the effective download delay for a domain.++ Takes the max of the spider's configured delay and any robots.txt+ directives (Crawl-delay / Request-rate). Result is cached per domain.+ Also pre-creates a per-domain concurrency limiter of 1 when robots.txt+ enforces any delay, before the caller acquires it via _rate_limiter().+ """+ robots_manager = self._robots_manager+ if robots_manager is None:+ return self.spider.download_delay++ domain = request.domain++ # Return cached delay if available+ if domain in self._domain_delays:+ return self._domain_delays[domain]++ # Fetch both robots.txt directives in a single parser lookup+ c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid)++ delay = self.spider.download_delay+ robots_enforced_delay = False++ if r_rate:+ req_count, period = r_rate+ if req_count > 0:+ delay = max(delay, period / req_count)+ robots_enforced_delay = True++ if c_delay is not None:+ delay = max(delay, c_delay)+ robots_enforced_delay = True++ self._domain_delays[domain] = delay++ # Enforce 1 concurrent request for this domain when robots.txt adds a delay+ if robots_enforced_delay and delay > 0 and domain not in self._domain_limiters:+ if self.spider.concurrent_requests_per_domain:+ log.warning(+ f"robots.txt for {domain} enforces a delay, overriding"+ f" concurrent_requests_per_domain={self.spider.concurrent_requests_per_domain} with 1"+ )+ self._domain_limiters[domain] = CapacityLimiter(1)++ return delay+def _rate_limiter(self, domain: str) -> CapacityLimiter:"""Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""if self.spider.concurrent_requests_per_domain:- if domain not in self._domain_limiters:- self._domain_limiters[domain] = CapacityLimiter(self.spider.concurrent_requests_per_domain)- return self._domain_limiters[domain]- return self._global_limiter+ self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))+ return self._domain_limiters.get(domain, self._global_limiter)def _normalize_request(self, request: Request) -> None:"""Normalize request fields before enqueueing.@@ -87,9 +143,21 @@ class CrawlerEngine:async def _process_request(self, request: Request) -> None:"""Download and process a single request."""+ if self._robots_manager:+ can_fetch = await self._robots_manager.can_fetch(request.url, request.sid)+ if not can_fetch:+ self.stats.robots_disallowed_count += 1+ log.debug(f"Request disallowed by robots.txt: {request.url}")+ return+ # Must be called before _rate_limiter: may create CapacityLimiter(1) in _domain_limiters+ # when robots.txt enforces a delay, which _rate_limiter then picks up.+ delay = await self._get_domain_delay(request)+ else:+ delay = self.spider.download_delay+async with self._rate_limiter(request.domain):- if self.spider.download_delay:- await anyio.sleep(self.spider.download_delay)+ if delay:+ await anyio.sleep(delay)if request._session_kwargs.get("proxy"):self.stats.proxies.append(request._session_kwargs["proxy"])@@ -227,15 +295,19 @@ class CrawlerEngine:self._pause_requested = Falseself._force_stop = Falseself.stats = CrawlStats(start_time=anyio.current_time())+ self._domain_limiters.clear()+ self._domain_delays.clear()# Check for existing checkpointresuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else Falseself._last_checkpoint_time = anyio.current_time()async with self.session_manager:+ # Set stats from spider configurationself.stats.concurrent_requests = self.spider.concurrent_requestsself.stats.concurrent_requests_per_domain = self.spider.concurrent_requests_per_domainself.stats.download_delay = self.spider.download_delay+await self.spider.on_start(resuming=resuming)try:diff --git a/scrapling/spiders/result.py b/scrapling/spiders/result.pyindex 08a7658..b374152 100644--- a/scrapling/spiders/result.py+++ b/scrapling/spiders/result.py@@ -47,6 +47,7 @@ class CrawlStats:concurrent_requests_per_domain: int = 0failed_requests_count: int = 0offsite_requests_count: int = 0+ robots_disallowed_count: int = 0response_bytes: int = 0items_scraped: int = 0items_dropped: int = 0@@ -95,6 +96,7 @@ class CrawlStats:"sessions_requests_count": self.sessions_requests_count,"failed_requests_count": self.failed_requests_count,"offsite_requests_count": self.offsite_requests_count,+ "robots_disallowed_count": self.robots_disallowed_count,"blocked_requests_count": self.blocked_requests_count,"response_status_count": self.response_status_count,"response_bytes": self.response_bytes,diff --git a/scrapling/spiders/spider.py b/scrapling/spiders/spider.pyindex 4f38912..6aaa24f 100644--- a/scrapling/spiders/spider.py+++ b/scrapling/spiders/spider.py@@ -72,6 +72,9 @@ class Spider(ABC):start_urls: list[str] = []allowed_domains: Set[str] = set()+ # Robots.txt compliance+ robots_txt_obey: bool = False+# Concurrency settingsconcurrent_requests: int = 4concurrent_requests_per_domain: int = 0