🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
style(spiders robots feat): Adjustments for maintainability
Karim shoair854daac7941a
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 163101b..f395a70 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -7,9 +7,9 @@ from anyio import Path as AsyncPathfrom anyio import create_task_group, CapacityLimiter, create_memory_object_stream, EndOfStreamfrom scrapling.core.utils import log-from scrapling.spiders.request import Requestfrom scrapling.spiders.scheduler import Schedulerfrom scrapling.spiders.session import SessionManager+from scrapling.spiders.request import Request, Responsefrom scrapling.spiders.robotstxt import RobotsTxtManagerfrom scrapling.spiders.result import CrawlStats, ItemListfrom scrapling.spiders.checkpoint import CheckpointManager, CheckpointData@@ -44,7 +44,7 @@ class CrawlerEngine:if self.spider.robots_txt_obey:- async def _fetch_robots(url: str, sid: str):+ async def _fetch_robots(url: str, sid: str) -> Response:return await self.session_manager.fetch(Request(url, sid=sid))self._robots_manager: Optional[RobotsTxtManager] = RobotsTxtManager(_fetch_robots)@@ -101,7 +101,7 @@ class CrawlerEngine:# Domains discovered mid-crawl (not in start_urls/allowed_domains) will fetch here.# Two concurrent callbacks hitting the same new domain can each trigger a fetch;# the second write is a no-op in effect (same content), but the extra request is accepted.- c_delay, r_rate = await robots_manager._get_delay_directives(request.url, request.sid)+ c_delay, r_rate = await robots_manager.get_delay_directives(request.url, request.sid)delay = self.spider.download_delayrobots_enforced_delay = False@@ -133,6 +133,7 @@ class CrawlerEngine:"""Get or create a per-domain concurrency limiter if enabled, otherwise use the global limiter."""if self.spider.concurrent_requests_per_domain:self._domain_limiters.setdefault(domain, CapacityLimiter(self.spider.concurrent_requests_per_domain))+ # robots.txt-created limiters always apply even with `concurrent_requests_per_domain = 0` (if enabled)return self._domain_limiters.get(domain, self._global_limiter)def _normalize_request(self, request: Request) -> None:diff --git a/scrapling/spiders/robotstxt.py b/scrapling/spiders/robotstxt.pyindex c64c66e..867698b 100644--- a/scrapling/spiders/robotstxt.py+++ b/scrapling/spiders/robotstxt.py@@ -116,7 +116,7 @@ class RobotsTxtManager:return (rate.requests, rate.seconds)return None- async def _get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:+ async def get_delay_directives(self, url: str, sid: str) -> tuple[Optional[float], Optional[tuple[int, int]]]:"""Return both crawl-delay and request-rate in a single parser lookup.Args: