Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

refactor(spider): prefetch robots.txt from start_urls only

Karim shoair556a90f645f4
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index 3d75564..7729d62 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -275,27 +275,19 @@ class CrawlerEngine:
async def _prefetch_robots_txt(self) -> None:
"""Pre-warm the robots.txt cache before the crawl loop starts.
- Uses allowed_domains if configured (defaults to https since bare domains
- have no scheme), otherwise falls back to unique domains extracted from
- start_urls preserving the original scheme.
+ Extracts unique domains from start_urls, preserving the original scheme.
"""
- if not self._robots_manager:
+ if not self._robots_manager or not self.spider.start_urls:
return
- if self._allowed_domains:
- # allowed_domains are bare strings like "example.com", no scheme available
- seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]
- elif self.spider.start_urls:
- # Deduplicate by netloc, preserving the scheme from the first URL per domain
- seen: set[str] = set()
- seed_urls = []
- for url in self.spider.start_urls:
- parsed = urlparse(url)
- if parsed.netloc not in seen:
- seen.add(parsed.netloc)
- seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
- else:
- return
+ # Deduplicate by netloc, preserving the scheme from the first URL per domain
+ seen: set[str] = set()
+ seed_urls: list[str] = []
+ for url in self.spider.start_urls:
+ parsed = urlparse(url)
+ if parsed.netloc not in seen:
+ seen.add(parsed.netloc)
+ seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)