🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
refactor(spider): prefetch robots.txt from start_urls only
Karim shoair556a90f645f4
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex 3d75564..7729d62 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -275,27 +275,19 @@ class CrawlerEngine:async def _prefetch_robots_txt(self) -> None:"""Pre-warm the robots.txt cache before the crawl loop starts.- Uses allowed_domains if configured (defaults to https since bare domains- have no scheme), otherwise falls back to unique domains extracted from- start_urls preserving the original scheme.+ Extracts unique domains from start_urls, preserving the original scheme."""- if not self._robots_manager:+ if not self._robots_manager or not self.spider.start_urls:return- if self._allowed_domains:- # allowed_domains are bare strings like "example.com", no scheme available- seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]- elif self.spider.start_urls:- # Deduplicate by netloc, preserving the scheme from the first URL per domain- seen: set[str] = set()- seed_urls = []- for url in self.spider.start_urls:- parsed = urlparse(url)- if parsed.netloc not in seen:- seen.add(parsed.netloc)- seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")- else:- return+ # Deduplicate by netloc, preserving the scheme from the first URL per domain+ seen: set[str] = set()+ seed_urls: list[str] = []+ for url in self.spider.start_urls:+ parsed = urlparse(url)+ if parsed.netloc not in seen:+ seen.add(parsed.netloc)+ seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)