Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spider): only allocate _domain_delays when robots_txt_obey is enabled

Karim shoair070338cf248d
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 1924298..f1baf76 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -54,9 +54,11 @@ class CrawlerEngine:
self._global_limiter = CapacityLimiter(spider.concurrent_requests)
self._domain_limiters: dict[str, CapacityLimiter] = {}
- self._domain_delays: dict[str, float] = {}
self._allowed_domains: set[str] = spider.allowed_domains or set()
+ if self.spider.robots_txt_obey:
+ self._domain_delays: dict[str, float] = {}
+
self._active_tasks: int = 0
self._running: bool = False
self._items: ItemList = ItemList()
@@ -300,7 +302,8 @@ class CrawlerEngine:
self._force_stop = False
self.stats = CrawlStats(start_time=anyio.current_time())
self._domain_limiters.clear()
- self._domain_delays.clear()
+ if self._robots_manager:
+ self._domain_delays.clear()
# Check for existing checkpoint
resuming = (await self._restore_from_checkpoint()) if self._checkpoint_system_enabled else False