Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

feat(spiders): Add SitemapSpider

Karim shoairf7da15771bdb
diff --git a/scrapling/spiders/__init__.py b/scrapling/spiders/__init__.py
index 92eb2e9..b455076 100644
--- a/scrapling/spiders/__init__.py
+++ b/scrapling/spiders/__init__.py
@@ -4,6 +4,8 @@ from .scheduler import Scheduler
from .engine import CrawlerEngine
from .session import SessionManager
from .spider import Spider, SessionConfigurationError
+from .links import LinkExtractor
+from .templates import CrawlSpider, SitemapSpider, CrawlRule
from scrapling.engines.toolbelt.custom import Response
__all__ = [
@@ -15,4 +17,8 @@ __all__ = [
"SessionManager",
"Scheduler",
"Response",
+ "LinkExtractor",
+ "CrawlSpider",
+ "CrawlRule",
+ "SitemapSpider",
]
diff --git a/scrapling/spiders/templates/__init__.py b/scrapling/spiders/templates/__init__.py
index 31d3ad1..fa758c7 100644
--- a/scrapling/spiders/templates/__init__.py
+++ b/scrapling/spiders/templates/__init__.py
@@ -1,6 +1,8 @@
from .crawler import CrawlSpider, CrawlRule
+from .sitemap import SitemapSpider
__all__ = [
"CrawlSpider",
"CrawlRule",
+ "SitemapSpider",
]
diff --git a/scrapling/spiders/templates/crawler.py b/scrapling/spiders/templates/crawler.py
index 342528c..5c6aeaa 100644
--- a/scrapling/spiders/templates/crawler.py
+++ b/scrapling/spiders/templates/crawler.py
@@ -65,7 +65,7 @@ class CrawlSpider(Spider):
for rule in self.rules():
for url in rule.link_extractor.extract(response):
req = response.follow(url, callback=rule.callback)
- if rule.priority:
+ if rule.priority is not None:
req.priority = rule.priority
if rule.process_request is not None:
req = rule.process_request(req, response)
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.py
new file mode 100644
index 0000000..bda8d3f
--- /dev/null
+++ b/scrapling/spiders/templates/sitemap.py
@@ -0,0 +1,193 @@
+"""Sitemap template spider."""
+
+from dataclasses import dataclass, field
+from gzip import GzipFile
+from io import BytesIO
+from urllib.parse import urlsplit
+
+from lxml import etree
+from protego import Protego
+
+from scrapling.core._types import (
+ TYPE_CHECKING,
+ Any,
+ AsyncGenerator,
+ Dict,
+ List,
+ Optional,
+ Union,
+)
+from scrapling.spiders.links import LinkExtractor
+from scrapling.spiders.request import Request
+from scrapling.spiders.spider import Spider
+from scrapling.spiders.templates.crawler import CrawlRule
+
+if TYPE_CHECKING:
+ from scrapling.engines.toolbelt.custom import Response
+
+
+__all__ = ["SitemapSpider"]
+
+
+_GZIP_MAGIC = b"\x1f\x8b"
+_GUNZIP_MAX_SIZE = 64 * 1024 * 1024 # 64 MiB cap, defends against gzip bombs
+
+
+@dataclass
+class SitemapResult:
+ """Parsed sitemap body.
+
+ `urls` holds the entries from a `<urlset>`; `sitemaps` holds child sitemap
+ URLs from a `<sitemapindex>` (each of which is fetched recursively).
+ """
+
+ urls: List[str] = field(default_factory=list)
+ sitemaps: List[str] = field(default_factory=list)
+
+
+class SitemapSpider(Spider):
+ """A Spider that seeds a crawl from sitemap(s), and follows the rules.
+
+ Override `rules()` to return a list of `CrawlRule`s.
+
+ If there are no rules provided, all non-sitemap urls will be redirected to `parse()`, which must be overridden or it will raise `NotImplementedError`.
+
+ :cvar sitemap_urls: Explicit list of sitemap (or robots.txt) URLs to fetch.
+ :cvar sitemap_follow: `LinkExtractor` filtering which child sitemaps inside a
+ `<sitemapindex>` to descend into. ``None`` means descend into all.
+ :cvar sitemap_alternate_links: When enabled, alternate-language URLs are also
+ routed through `rules()`.
+ """
+
+ sitemap_urls: List[str] = []
+ sitemap_follow: Optional[LinkExtractor] = None
+ sitemap_alternate_links: bool = False
+
+ def rules(self) -> List[CrawlRule]:
+ """Override to define dispatch rules for sitemap URLs."""
+ return []
+
+ async def start_requests(self) -> AsyncGenerator[Request, None]:
+ if self.sitemap_urls:
+ for url in self.sitemap_urls:
+ yield Request(url, callback=self._parse_sitemap)
+ return
+
+ raise RuntimeError("`SitemapSpider` needs `sitemap_urls` to be set.")
+
+ async def parse(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:
+ """Default callback for processing responses"""
+ raise NotImplementedError(f"{self.__class__.__name__} must implement parse() method")
+ yield # Make this a generator for type checkers
+
+ def _robots_body(self, response: "Response") -> List[str]:
+ """Extract `Sitemap` directives from a robots.txt body via protego."""
+ try:
+ text = response.body.decode(response.encoding, errors="replace")
+ parser = Protego.parse(text)
+ except Exception as e:
+ self.logger.warning(f"Failed to parse robots.txt: {e}")
+ return []
+ return list(parser.sitemaps)
+
+ @staticmethod
+ def _decompress(body: bytes, content_type: Optional[str]) -> bytes:
+ if (content_type and ("gzip" in content_type.lower())) or (body[:2] == _GZIP_MAGIC):
+ out = bytearray()
+ with GzipFile(fileobj=BytesIO(body)) as f:
+ while chunk := f.read1(8192):
+ out.extend(chunk)
+ if len(out) > _GUNZIP_MAX_SIZE:
+ raise OSError(f"gzip output exceeds {_GUNZIP_MAX_SIZE} bytes")
+ return bytes(out)
+ return body
+
+ def _extract_urls(self, root: Any) -> List[str]:
+ urls: List[str] = []
+ for url_el in root:
+ if self._get_type(url_el) != "url":
+ continue
+
+ for child in url_el:
+ name = self._get_type(child)
+ if name == "loc" and child.text:
+ urls.append(child.text.strip())
+ elif self.sitemap_alternate_links and name == "link":
+ href = child.get("href")
+ if href:
+ urls.append(href.strip())
+ return urls
+
+ @staticmethod
+ def _get_type(el: Any) -> str:
+ return etree.QName(el.tag).localname
+
+ def _sm_body(self, body: bytes, content_type: Optional[str] = None) -> SitemapResult:
+ """Parse a sitemap body and return its URLs and any child sitemaps."""
+ try:
+ body = self._decompress(body, content_type)
+ except OSError as e:
+ self.logger.warning(f"Failed to decompress sitemap: {e}")
+ return SitemapResult()
+
+ try:
+ root = etree.fromstring(body)
+ except etree.XMLSyntaxError as e:
+ self.logger.warning(f"Failed to parse sitemap XML: {e}")
+ return SitemapResult()
+
+ root_name = self._get_type(root)
+ if root_name == "sitemapindex":
+ locs = []
+ for sm_el in root:
+ if self._get_type(sm_el) == "sitemap":
+ for child in sm_el:
+ if self._get_type(child) == "loc" and child.text:
+ locs.append(child.text.strip())
+ break
+ return SitemapResult(sitemaps=locs)
+ if root_name == "urlset":
+ return SitemapResult(urls=self._extract_urls(root))
+
+ self.logger.warning(f"Unknown sitemap root element: {root_name!r}")
+ return SitemapResult()
+
+ async def _parse_sitemap(self, response: "Response") -> AsyncGenerator[Union[Dict[str, Any], Request, None], None]:
+ if urlsplit(response.url).path.endswith("/robots.txt"):
+ sitemaps = self._robots_body(response)
+ if not sitemaps:
+ self.logger.warning(f"No Sitemaps found in {response.url}")
+
+ for sitemap_url in sitemaps:
+ yield response.follow(sitemap_url, callback=self._parse_sitemap)
+ return
+
+ content_type = response.headers.get("content-type") if response.headers else None
+ result = self._sm_body(response.body, content_type=content_type)
+
+ # Descend into child sitemaps (apply sitemap_follow filter if present)
+ for child_url in result.sitemaps:
+ if self.sitemap_follow is not None and not self.sitemap_follow.matches(child_url):
+ continue
+ yield response.follow(child_url, callback=self._parse_sitemap)
+
+ # Dispatch each URL through rules() (first match wins; unmatched drop unless rules empty)
+ rules = self.rules()
+ for url in result.urls:
+ req = self._dispatch(response, url, rules)
+ if req is not None:
+ yield req
+
+ @staticmethod
+ def _dispatch(response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]:
+ if not rules:
+ return response.follow(url)
+ for rule in rules:
+ if rule.link_extractor.matches(url):
+ req = response.follow(url, callback=rule.callback)
+ if rule.priority is not None:
+ req.priority = rule.priority
+ if rule.process_request is not None:
+ req = rule.process_request(req, response)
+ return req
+ return None