Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spider robots): solve multiple issues with cache prefetch

Karim shoairaf83a11aa787
diff --git a/scrapling/spiders/engine.py b/scrapling/spiders/engine.py
index f395a70..3b41fb5 100644
--- a/scrapling/spiders/engine.py
+++ b/scrapling/spiders/engine.py
@@ -1,6 +1,7 @@
import json
import pprint
from pathlib import Path
+from urllib.parse import urlparse
import anyio
from anyio import Path as AsyncPath
@@ -294,22 +295,28 @@ class CrawlerEngine:
async def _prefetch_robots_txt(self) -> None:
"""Pre-warm the robots.txt cache before the crawl loop starts.
- Uses allowed_domains if configured, otherwise falls back to unique domains
- extracted from start_urls via Request.domain. Both paths use https.
+ Uses allowed_domains if configured (defaults to https since bare domains
+ have no scheme), otherwise falls back to unique domains extracted from
+ start_urls preserving the original scheme.
"""
if not self._robots_manager:
return
if self._allowed_domains:
- domains = self._allowed_domains
+ # allowed_domains are bare strings like "example.com", no scheme available
+ seed_urls = [f"http://{domain}/" for domain in self._allowed_domains]
elif self.spider.start_urls:
- # Deduplicate by domain so we spawn exactly one task per domain
- domains = {Request(url).domain for url in self.spider.start_urls}
+ # Deduplicate by netloc, preserving the scheme from the first URL per domain
+ seen: set[str] = set()
+ seed_urls = []
+ for url in self.spider.start_urls:
+ parsed = urlparse(url)
+ if parsed.netloc not in seen:
+ seen.add(parsed.netloc)
+ seed_urls.append(f"{parsed.scheme}://{parsed.netloc}/")
else:
return
- seed_urls = [f"https://{domain}/" for domain in domains]
-
await self._robots_manager.prefetch(seed_urls, self.session_manager.default_session_id)
async def crawl(self) -> CrawlStats: