🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spider): only allocate _domain_delays when robots_txt_obey is enabled
Karim shoair070338cf248d
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 1924298..f1baf76 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -54,9 +54,11 @@ class CrawlerEngine:self._global_limiter = CapacityLimiter(spider.concurrent_requests)self._domain_limiters: dict[str, CapacityLimiter] = {}- self._domain_delays: dict[str, float] = {}self._allowed_domains: set[str] = spider.allowed_domains or set()+ if self.spider.robots_txt_obey:+ self._domain_delays: dict[str, float] = {}+self._active_tasks: int = 0self._running: bool = Falseself._items: ItemList = ItemList()@@ -300,7 +302,8 @@ class CrawlerEngine:self._force_stop = Falseself.stats = CrawlStats(start_time=anyio.current_time())self._domain_limiters.clear()- self._domain_delays.clear()+ if self._robots_manager:+ self._domain_delays.clear()# Check for existing checkpointresuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False