🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spider robots): solve multiple issues with cache prefetch
Karim shoairaf83a11aa787
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.pyindex f395a70..3b41fb5 100644--- a/scrapling/spiders/engine.py+++ b/scrapling/spiders/engine.py@@ -1,6 +1,7 @@import jsonimport pprintfrom pathlib import Path+from urllib.parse import urlparseimport anyiofrom anyio import Path as AsyncPath@@ -294,22 +295,28 @@ class CrawlerEngine:async def _prefetch_robots_txt(self) -> None:"""Pre-warm the robots.txt cache before the crawl loop starts.- Uses allowed_domains if configured, otherwise falls back to unique domains- extracted from start_urls via Request.domain. Both paths use https.+ Uses allowed_domains if configured (defaults to https since bare domains+ have no scheme), otherwise falls back to unique domains extracted from+ start_urls preserving the original scheme."""if not self._robots_manager:returnif self._allowed_domains:- domains = self._allowed_domains+ # allowed_domains are bare strings like "example.com", no scheme available+ seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]elif self.spider.start_urls:- # Deduplicate by domain so we spawn exactly one task per domain- domains = {Request(url).domain for url in self.spider.start_urls}+ # Deduplicate by netloc, preserving the scheme from the first URL per domain+ seen: set[str] = set()+ seed_urls = []+ for url in self.spider.start_urls:+ parsed = urlparse(url)+ if parsed.netloc not in seen:+ seen.add(parsed.netloc)+ seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")else:return- seed_urls = [f"https://{domain}/" for domain in domains]-await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)async def crawl(self) -> CrawlStats: