Scrapy, a fast high-level web crawling & scraping framework for Python.
Resolve relative and protocol-relative sitemap locs (#8210)
Adrian7ca2fe550c3c
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 4e73d247d..36608a87b 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -6,6 +6,7 @@ import re# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotationfrom collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003from typing import TYPE_CHECKING, Any, cast+from urllib.parse import urljoinfrom scrapy.http import Request, Response, XmlResponsefrom scrapy.spiders import Spider@@ -86,12 +87,16 @@ class SitemapSpider(Spider):s = Sitemap(body)if s.type == "sitemapindex":- urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))+ urls = list(+ self._get_urls_from_sitemapindex(self.sitemap_filter(s), response.url)+ )return (Request(loc, callback=self._parse_sitemap) for loc in urls)if s.type == "urlset":url_callback_pairs = list(- self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))+ self._get_urls_and_callbacks_from_urlset(+ self.sitemap_filter(s), response.url+ ))return (Request(loc, callback=c) for loc, c in url_callback_pairs)@@ -104,16 +109,18 @@ class SitemapSpider(Spider):return ()def _get_urls_from_sitemapindex(- self, it: Iterable[dict[str, Any]]+ self, it: Iterable[dict[str, Any]], base_url: str) -> Iterable[str]:for loc in iterloc(it, self.sitemap_alternate_links):+ loc = urljoin(base_url, loc) # noqa: PLW2901if any(x.search(loc) for x in self._follow):yield locdef _get_urls_and_callbacks_from_urlset(- self, it: Iterable[dict[str, Any]]+ self, it: Iterable[dict[str, Any]], base_url: str) -> Iterable[tuple[str, CallbackT]]:for loc in iterloc(it, self.sitemap_alternate_links):+ loc = urljoin(base_url, loc) # noqa: PLW2901for r, c in self._cbs:if r.search(loc):yield loc, cdiff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex 000c2f9d7..0072d9f28 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -152,6 +152,19 @@ Sitemap: /sitemap-relative-url.xml"http://www.example.com/italiano/",]+ def test_relative_and_protocol_relative_locs(self):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url><loc>//www.example.com/protocol-relative/</loc></url>+ <url><loc>/relative/</loc></url>+ </urlset>"""+ r = TextResponse(url="https://www.example.com/sitemap.xml", body=sitemap)+ spider = self.spider_class("example.com")+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "https://www.example.com/protocol-relative/",+ "https://www.example.com/relative/",+ ]+def test_sitemap_filter(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"