🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev
Python aiai-scrapingautomationcrawlercrawlingcrawling-pythondatadata-extractionmcpmcp-serverplaywrightpythonscrapingselectorsstealthweb-scraperweb-scrapingweb-scraping-pythonwebscrapingxpath
fix(spiders): route sitemap URLs to parse() instead of the sitemap parser (#458)
Suliman Abdulrazzaq8d600c4edd6d
diff --git a/scrapling/spiders/templates/sitemap.py b/scrapling/spiders/templates/sitemap.pyindex a16db62..20e21f3 100644--- a/scrapling/spiders/templates/sitemap.py+++ b/scrapling/spiders/templates/sitemap.py@@ -164,13 +164,13 @@ class SitemapSpider(Spider):if req is not None:yield req- @staticmethod- def _dispatch(response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]:+ def _dispatch(self, response: "Response", url: str, rules: List[CrawlRule]) -> Optional[Request]:+ # The callback is always set explicitly, otherwise `follow()` would inherit `_parse_sitemap` from the sitemap requestif not rules:- return response.follow(url)+ return response.follow(url, callback=self.parse)for rule in rules:if rule.link_extractor.matches(url):- req = response.follow(url, callback=rule.callback)+ req = response.follow(url, callback=rule.callback or self.parse)if rule.priority is not None:req.priority = rule.priorityif rule.process_request is not None:diff --git a/tests/spiders/test_sitemap.py b/tests/spiders/test_sitemap.pyindex 725fff6..b8b709f 100644--- a/tests/spiders/test_sitemap.py+++ b/tests/spiders/test_sitemap.py@@ -109,7 +109,49 @@ class TestSitemapSpiderFlow:spider = S()out = await _collect(spider._parse_sitemap(_make_response(URLSET_XML)))assert len(out) == 3- assert all(r.callback is None for r in out)+ assert all(r.callback == spider.parse for r in out)++ @pytest.mark.asyncio+ async def test_no_rules_urls_go_to_parse_not_sitemap_parser(self):+ """In a real crawl, the sitemap response's request carries `_parse_sitemap` as its callback,+ which the page requests must not inherit."""++ class S(SitemapSpider):+ name = "s"+ sitemap_urls = ["https://example.com/sitemap.xml"]++ spider = S()+ response = _make_response(URLSET_XML)+ response.request = Request(response.url, sid="default", callback=spider._parse_sitemap)+ out = await _collect(spider._parse_sitemap(response))+ assert len(out) == 3+ assert all(r.callback == spider.parse for r in out)++ @pytest.mark.asyncio+ async def test_rule_without_callback_goes_to_parse_not_sitemap_parser(self):+ class S(SitemapSpider):+ name = "s"+ sitemap_urls = ["https://example.com/sitemap.xml"]++ def rules(self):+ return [+ CrawlRule(LinkExtractor(allow=r"/posts/"), callback=self.parse_post),+ CrawlRule(LinkExtractor(allow=r"/about")),+ ]++ async def parse_post(self, response):+ yield {"post": response.url}++ spider = S()+ response = _make_response(URLSET_XML)+ response.request = Request(response.url, sid="default", callback=spider._parse_sitemap)+ out = await _collect(spider._parse_sitemap(response))+ callbacks = {r.url: r.callback for r in out}+ assert callbacks == {+ "https://example.com/posts/1": spider.parse_post,+ "https://example.com/posts/2": spider.parse_post,+ "https://example.com/about": spider.parse,+ }@pytest.mark.asyncioasync def test_comments_in_sitemap_are_skipped(self):