🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spider): Make delay in robots file don't affect user's concurrency settings
Karim shoairec487d37e87f
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 3b41fb5..3d75564 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -85,8 +85,6 @@ class CrawlerEngine:Takes the max of the spider's configured delay and any robots.txtdirectives (Crawl-delay / Request-rate). Result is cached per domain.- Also pre-creates a per-domain concurrency limiter of 1 when robots.txt- enforces any delay, before the caller acquires it via _rate_limiter()."""robots_manager = self._robots_managerif robots_manager is None:@@ -94,48 +92,32 @@ class CrawlerEngine:domain = request.domain- # Return cached delay if availableif domain in self._domain_delays:return self._domain_delays[domain]# For domains covered by _prefetch_robots_txt this is a local parser read.# Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.- # Two concurrent callbacks hitting the same new domain can each trigger a fetch;- # the second write is a no-op in effect (same content), but the extra request is accepted.c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)delay = self.spider.download_delay- robots_enforced_delay = Falseif r_rate:req_count, period = r_rateif req_count > 0:delay = max(delay, period / req_count)- robots_enforced_delay = Trueif c_delay is not None:delay = max(delay, c_delay)- robots_enforced_delay = Trueself._domain_delays[domain] = delay-- # Enforce 1 concurrent request for this domain when robots.txt adds a delay- if robots_enforced_delay and delay > 0 and domain not in self._domain_limiters:- if self.spider.concurrent_requests_per_domain:- log.warning(- f"robots.txt for {domain} enforces a delay, overriding"- f" concurrent_requests_per_domain={self.spider.concurrent_requests_per_domain} with 1"- )- self._domain_limiters[domain] = CapacityLimiter(1)-return delaydef _rate_limiter(self, domain: str) -> CapacityLimiter:"""Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""if self.spider.concurrent_requests_per_domain:self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))- # robots.txt-created limiters always apply even with `concurrent_requests_per_domain = 0` (if enabled)- return self._domain_limiters.get(domain, self._global_limiter)+ return self._domain_limiters[domain]+ return self._global_limiterdef _normalize_request(self, request: Request) -> None:"""Normalize request fields before enqueueing.@@ -154,8 +136,6 @@ class CrawlerEngine:self.stats.robots_disallowed_count += 1log.debug(f"Request disallowed by robots.txt: {request.url}")return- # Must be called before _rate_limiter: may create CapacityLimiter(1) in _domain_limiters- # when robots.txt enforces a delay, which _rate_limiter then picks up.delay = await self._get_domain_delay(request)else:delay = self.spider.download_delay