Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

fix(spiders): route sitemap URLs to parse() instead of the sitemap parser (#458)

Suliman Abdulrazzaq8d600c4edd6d
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.py
index a16db62..20e21f3 100644
--- a/scrapling/spiders/templates/sitemap.py
+++ b/scrapling/spiders/templates/sitemap.py
@@ -164,13 +164,13 @@ class SitemapSpider(Spider):
if req is not None:
yield req
- @staticmethod
- def _dispatch(response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]:
+ def _dispatch(self, response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]:
+ # The callback is always set explicitly, otherwise `follow()` would inherit `_parse_sitemap` from the sitemap request
if not rules:
- return response.follow(url)
+ return response.follow(url, callback=self.parse)
for rule in rules:
if rule.link_extractor.matches(url):
- req = response.follow(url, callback=rule.callback)
+ req = response.follow(url, callback=rule.callback or self.parse)
if rule.priority is not None:
req.priority = rule.priority
if rule.process_request is not None:
diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.py
index 725fff6..b8b709f 100644
--- a/tests/spiders/test_sitemap.py
+++ b/tests/spiders/test_sitemap.py
@@ -109,7 +109,49 @@ class TestSitemapSpiderFlow:
spider = S()
out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML)))
assert len(out) == 3
- assert all(r.callback is None for r in out)
+ assert all(r.callback == spider.parse for r in out)
+
+ @pytest.mark.asyncio
+ async def test_no_rules_urls_go_to_parse_not_sitemap_parser(self):
+ """In a real crawl, the sitemap response's request carries `_parse_sitemap` as its callback,
+ which the page requests must not inherit."""
+
+ class S(SitemapSpider):
+ name = "s"
+ sitemap_urls = ["https://example.com/sitemap.xml"]
+
+ spider = S()
+ response = _make_response(URLSET_XML)
+ response.request = Request(response.url, sid="default", callback=spider._parse_sitemap)
+ out = await _collect(spider._parse_sitemap(response))
+ assert len(out) == 3
+ assert all(r.callback == spider.parse for r in out)
+
+ @pytest.mark.asyncio
+ async def test_rule_without_callback_goes_to_parse_not_sitemap_parser(self):
+ class S(SitemapSpider):
+ name = "s"
+ sitemap_urls = ["https://example.com/sitemap.xml"]
+
+ def rules(self):
+ return [
+ CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post),
+ CrawlRule(LinkExtractor(allow=r"/about")),
+ ]
+
+ async def parse_post(self, response):
+ yield {"post": response.url}
+
+ spider = S()
+ response = _make_response(URLSET_XML)
+ response.request = Request(response.url, sid="default", callback=spider._parse_sitemap)
+ out = await _collect(spider._parse_sitemap(response))
+ callbacks = {r.url: r.callback for r in out}
+ assert callbacks == {
+ "https://example.com/posts/1": spider.parse_post,
+ "https://example.com/posts/2": spider.parse_post,
+ "https://example.com/about": spider.parse,
+ }
@pytest.mark.asyncio
async def test_comments_in_sitemap_are_skipped(self):