Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Resolve relative and protocol-relative sitemap locs (#8210)

Adrian7ca2fe550c3c
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 4e73d247d..36608a87b 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -6,6 +6,7 @@ import re
# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003
from typing import TYPE_CHECKING, Any, cast
+from urllib.parse import urljoin
from scrapy.http import Request, Response, XmlResponse
from scrapy.spiders import Spider
@@ -86,12 +87,16 @@ class SitemapSpider(Spider):
s = Sitemap(body)
if s.type == "sitemapindex":
- urls = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s)))
+ urls = list(
+ self._get_urls_from_sitemapindex(self.sitemap_filter(s), response.url)
+ )
return (Request(loc, callback=self._parse_sitemap) for loc in urls)
if s.type == "urlset":
url_callback_pairs = list(
- self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s))
+ self._get_urls_and_callbacks_from_urlset(
+ self.sitemap_filter(s), response.url
+ )
)
return (Request(loc, callback=c) for loc, c in url_callback_pairs)
@@ -104,16 +109,18 @@ class SitemapSpider(Spider):
return ()
def _get_urls_from_sitemapindex(
- self, it: Iterable[dict[str, Any]]
+ self, it: Iterable[dict[str, Any]], base_url: str
) -> Iterable[str]:
for loc in iterloc(it, self.sitemap_alternate_links):
+ loc = urljoin(base_url, loc) # noqa: PLW2901
if any(x.search(loc) for x in self._follow):
yield loc
def _get_urls_and_callbacks_from_urlset(
- self, it: Iterable[dict[str, Any]]
+ self, it: Iterable[dict[str, Any]], base_url: str
) -> Iterable[tuple[str, CallbackT]]:
for loc in iterloc(it, self.sitemap_alternate_links):
+ loc = urljoin(base_url, loc) # noqa: PLW2901
for r, c in self._cbs:
if r.search(loc):
yield loc, c
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 000c2f9d7..0072d9f28 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -152,6 +152,19 @@ Sitemap: /sitemap-relative-url.xml
"http://www.example.com/italiano/",
]
+ def test_relative_and_protocol_relative_locs(self):
+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url><loc>//www.example.com/protocol-relative/</loc></url>
+ <url><loc>/relative/</loc></url>
+ </urlset>"""
+ r = TextResponse(url="https://www.example.com/sitemap.xml", body=sitemap)
+ spider = self.spider_class("example.com")
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "https://www.example.com/protocol-relative/",
+ "https://www.example.com/relative/",
+ ]
+
def test_sitemap_filter(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"