Scrapy, a fast high-level web crawling & scraping framework for Python.
Add SitemapSpider.sitemap_request() to build requests from sitemap entries (#8175)
Adrianc38f20cf3b53
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex e05a4518d..3164dbe75 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -607,10 +607,13 @@ robots_parsed~~~~~~~~~~~~~.. signal:: robots_parsed-.. function:: robots_parsed(robotparser, request)+.. function:: robots_parsed(robotparser, request, response).. versionadded:: 2.18.0+ .. versionchanged:: VERSION+ Added *response*.+Sent by:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after itdownloads and parses a :file:`robots.txt` file, for the host that *request*@@ -624,6 +627,9 @@ robots_parsed:param request: the request that triggered the :file:`robots.txt` download:type request: :class:`~scrapy.Request` object+ :param response: the :file:`robots.txt` response+ :type response: :class:`~scrapy.http.Response` object+Response signals----------------diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 3d4a537fd..a0525af79 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -218,6 +218,7 @@ class RobotsTxtMiddleware:signal=signals.robots_parsed,robotparser=rp,request=request,+ response=response,)rp_dfd = self._parsers[netloc]assert isinstance(rp_dfd, Deferred)diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 0c14a4cc6..a54324b00 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -2,13 +2,15 @@ from __future__ import annotationsimport loggingimport re+import warnings# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotationfrom collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003-from typing import TYPE_CHECKING, Any, Self, cast+from typing import TYPE_CHECKING, Any, Literal, Self, castfrom urllib.parse import urljoinfrom scrapy import signals+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Response, XmlResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider@@ -69,9 +71,10 @@ class SitemapSpider(Spider):"""sitemap_follow: Sequence[re.Pattern[str] | str] = [""]- """Regexes of sitemaps that should be followed. This is only for sites that- use `sitemap index files <https://www.sitemaps.org/protocol.html#index>`_- that point to other sitemap files.+ """Regexes of sitemap URLs that should be followed: those linked from+ `sitemap index files <https://www.sitemaps.org/protocol.html#index>`_, and+ those declared through the ``Sitemap`` directive of a :file:`robots.txt`+ file.By default, all sitemaps are followed."""@@ -123,7 +126,9 @@ class SitemapSpider(Spider):self._robots_sitemaps: dict[str, list[str]] = {}self._signal_only_netlocs: set[str] = set()- def _robots_parsed(self, robotparser: RobotParser, request: Request) -> None:+ def _robots_parsed(+ self, robotparser: RobotParser, request: Request, response: Response+ ) -> None:netloc = urlparse_cached(request).netlocurls = [urljoin(request.url, url) for url in robotparser.sitemaps()]if netloc in self._signal_only_netlocs:@@ -132,7 +137,16 @@ class SitemapSpider(Spider):# directly instead of stashing them for a response that never arrives.self._signal_only_netlocs.discard(netloc)for url in urls:- self.crawler.engine.crawl(Request(url, callback=self._parse_sitemap))+ if any(x.search(url) for x in self._follow):+ self.crawler.engine.crawl(+ self.sitemap_request(+ url,+ self._parse_sitemap,+ {"loc": url},+ source="robotstxt",+ response=response,+ )+ )else:self._robots_sitemaps[netloc] = urls@@ -211,6 +225,45 @@ class SitemapSpider(Spider):"""yield from entries+ def sitemap_request(+ self,+ loc: str,+ callback: CallbackT,+ entry: dict[str, Any],+ *,+ source: Literal["robotstxt", "sitemapindex", "urlset"],+ response: Response,+ ) -> Request:+ """Return the request for the *loc* URL of the sitemap *entry*, to be+ parsed with *callback*.++ .. versionadded:: VERSION++ Override this method to build those requests differently, e.g. to+ carry sitemap data and request metadata over to *callback*:++ .. code-block:: python++ def sitemap_request(self, loc, callback, entry, *, source, response):+ meta = {+ "lastmod": entry.get("lastmod"),+ "site": response.meta.get("site"),+ }+ return Request(loc, callback, meta=meta)++ *entry* is the sitemap entry that *loc* comes from, as described in+ :meth:`sitemap_filter`. When :attr:`sitemap_alternate_links` is+ enabled, the same entry is used for every alternate URL.++ *source* tells where *loc* comes from: ``"urlset"`` for a URL to+ crawl, ``"sitemapindex"`` for a sitemap linked from a sitemap index,+ and ``"robotstxt"`` for a sitemap linked from a :file:`robots.txt`+ file, where *entry* only has ``loc``.++ *response* is the response that *entry* was read from.+ """+ return Request(loc, callback=callback)+def _parse_sitemap(self, response: Response) -> Iterable[Request]:if response.url.endswith("/robots.txt"):urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))@@ -218,7 +271,17 @@ class SitemapSpider(Spider):for url in self._robots_sitemaps.pop(netloc, ()):if url not in urls:urls.append(url)- return (Request(url, callback=self._parse_sitemap) for url in urls)+ return (+ self.sitemap_request(+ url,+ self._parse_sitemap,+ {"loc": url},+ source="robotstxt",+ response=response,+ )+ for url in urls+ if any(x.search(url) for x in self._follow)+ )body = self._get_sitemap_body(response)if not body:@@ -232,18 +295,30 @@ class SitemapSpider(Spider):s = Sitemap(body)if s.type == "sitemapindex":- urls = list(+ index_entries = list(self._get_urls_from_sitemapindex(self.sitemap_filter(s), response.url))- return (Request(loc, callback=self._parse_sitemap) for loc in urls)+ return (+ self.sitemap_request(+ loc,+ self._parse_sitemap,+ entry,+ source="sitemapindex",+ response=response,+ )+ for loc, entry in index_entries+ )if s.type == "urlset":- url_callback_pairs = list(+ urlset_entries = list(self._get_urls_and_callbacks_from_urlset(self.sitemap_filter(s), response.url))- return (Request(loc, callback=c) for loc, c in url_callback_pairs)+ return (+ self.sitemap_request(loc, c, entry, source="urlset", response=response)+ for loc, c, entry in urlset_entries+ )logger.warning("Ignoring invalid sitemap: %(response)s",@@ -255,20 +330,20 @@ class SitemapSpider(Spider):def _get_urls_from_sitemapindex(self, it: Iterable[dict[str, Any]], base_url: str- ) -> Iterable[str]:- for loc in iterloc(it, self.sitemap_alternate_links):+ ) -> Iterable[tuple[str, dict[str, Any]]]:+ for loc, entry in _iterlocs(it, self.sitemap_alternate_links):loc = urljoin(base_url, loc) # noqa: PLW2901if any(x.search(loc) for x in self._follow):- yield loc+ yield loc, entrydef _get_urls_and_callbacks_from_urlset(self, it: Iterable[dict[str, Any]], base_url: str- ) -> Iterable[tuple[str, CallbackT]]:- for loc in iterloc(it, self.sitemap_alternate_links):+ ) -> Iterable[tuple[str, CallbackT, dict[str, Any]]]:+ for loc, entry in _iterlocs(it, self.sitemap_alternate_links):loc = urljoin(base_url, loc) # noqa: PLW2901for r, c in self._cbs:if r.search(loc):- yield loc, c+ yield loc, c, entrybreakdef _get_sitemap_body(self, response: Response) -> bytes | None:@@ -324,10 +399,22 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]:def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]:+ warnings.warn(+ "scrapy.spiders.sitemap.iterloc() is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return (loc for loc, _ in _iterlocs(it, alt))+++def _iterlocs(+ it: Iterable[dict[str, Any]], alt: bool = False+) -> Iterable[tuple[str, dict[str, Any]]]:for d in it:if loc := d["loc"]:- yield loc+ yield loc, d# Also consider alternate URLs (xhtml:link rel="alternate")if alt and (alt_list := d.get("alternate")):- yield from alt_list+ for loc in alt_list:+ yield loc, ddiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex c7e11ccef..ab809b387 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -96,6 +96,7 @@ Disallow: /some/randome/page.htmlassert len(calls) == 1assert calls[0]["request"] is requestassert calls[0]["robotparser"] is not None+ assert calls[0]["response"].url == "http://site.local/robots.txt"@coroutine_testasync def test_robotstxt_multiple_reqs(self) -> None:diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex da5b2b24d..ca5eb0985 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -12,8 +12,10 @@ from unittest import mockimport pytestfrom scrapy import signals+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponsefrom scrapy.spiders import SitemapSpider+from scrapy.spiders.sitemap import iterlocfrom scrapy.utils.test import get_crawlerfrom tests import tests_datadirfrom tests.spiders import RawResponseSpider@@ -164,6 +166,7 @@ Sitemap: /sitemap-relative-url.xmlsignal=signals.robots_parsed,robotparser=FakeRobotParser(),request=Request("http://www.example.com/robots.txt"),+ response=TextResponse("http://www.example.com/robots.txt"),)robots = b"Sitemap: http://example.com/sitemap.xml\n"@@ -181,13 +184,23 @@ Sitemap: /sitemap-relative-url.xmlsignal=signals.robots_parsed,robotparser=None,request=Request("http://www.example.com/robots.txt"),+ response=TextResponse("http://www.example.com/robots.txt"),)assert spider._robots_sitemaps == {}def test_robots_parsed_signal_schedules_sitemaps_for_skipped_netloc(self):- crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})- spider = self.spider_class.from_crawler(crawler, "example.com")+ calls = []++ class _Spider(self.spider_class): # type: ignore[name-defined,misc]+ def sitemap_request(self, loc, callback, entry, *, source, response):+ calls.append((loc, entry, source, response))+ return super().sitemap_request(+ loc, callback, entry, source=source, response=response+ )++ crawler = get_crawler(_Spider, settings_dict={"ROBOTSTXT_OBEY": True})+ spider = _Spider.from_crawler(crawler, "example.com")crawler.engine = mock.MagicMock()spider._signal_only_netlocs.add("www.example.com")@@ -195,21 +208,49 @@ Sitemap: /sitemap-relative-url.xmldef sitemaps(self):return ["/sitemap.xml"]+ robots_response = TextResponse("http://www.example.com/robots.txt")# request is whatever other request to that host triggered the fetch,# not necessarily a request for robots.txt itself.crawler.signals.send_catch_log(signal=signals.robots_parsed,robotparser=FakeRobotParser(),request=Request("http://www.example.com/some/page"),+ response=robots_response,)crawler.engine.crawl.assert_called_once()scheduled = crawler.engine.crawl.call_args[0][0]assert scheduled.url == "http://www.example.com/sitemap.xml"assert scheduled.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable+ url = "http://www.example.com/sitemap.xml"+ assert calls == [(url, {"loc": url}, "robotstxt", robots_response)]assert spider._signal_only_netlocs == set()assert spider._robots_sitemaps == {}+ def test_sitemap_follow_robots_parsed_signal(self):+ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]+ sitemap_follow = [r"1\.xml"]++ crawler = get_crawler(_FollowSpider, settings_dict={"ROBOTSTXT_OBEY": True})+ spider = _FollowSpider.from_crawler(crawler, "example.com")+ crawler.engine = mock.MagicMock()+ spider._signal_only_netlocs.add("www.example.com")++ class FakeRobotParser:+ def sitemaps(self):+ return ["/sitemap1.xml", "/sitemap2.xml"]++ crawler.signals.send_catch_log(+ signal=signals.robots_parsed,+ robotparser=FakeRobotParser(),+ request=Request("http://www.example.com/some/page"),+ response=TextResponse("http://www.example.com/robots.txt"),+ )++ crawler.engine.crawl.assert_called_once()+ scheduled = crawler.engine.crawl.call_args[0][0]+ assert scheduled.url == "http://www.example.com/sitemap1.xml"+def test_alternate_url_locs(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"@@ -561,6 +602,154 @@ Sitemap: /sitemap-relative-url.xmlurls = [req.url for req in spider._parse_sitemap(r)]assert urls == result+ @pytest.mark.parametrize(+ ("follow", "result"),+ [+ (r"1.xml", ["http://www.example.com/sitemap1.xml"]),+ (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]),+ (r"nonexistent", []),+ ],+ )+ def test_sitemap_follow_robotstxt(self, follow, result):+ robots = b"Sitemap: http://www.example.com/sitemap1.xml\n"+ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)++ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]+ sitemap_follow = [follow]++ spider = _FollowSpider("example.com")+ urls = [req.url for req in spider._parse_sitemap(r)]+ assert urls == result++ @coroutine_test+ async def test_sitemap_request(self, mockserver: MockServer):+ class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]+ def sitemap_request(self, loc, callback, entry, *, source, response):+ return Request(loc, callback, meta={"lastmod": entry["lastmod"]})++ def parse(self, response):+ yield {"url": response.url, "lastmod": response.meta["lastmod"]}++ def raw_body(self):+ assert self.mockserver+ loc = self.mockserver.url("/text")+ return (+ '<?xml version="1.0" encoding="UTF-8"?>'+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'+ f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod></url>"+ "</urlset>"+ )++ items, _ = await crawl_items(_Spider, mockserver)+ assert items == [{"url": mockserver.url("/text"), "lastmod": "2005-01-01"}]++ @pytest.mark.parametrize(+ ("url", "body", "expected"),+ [+ (+ "http://www.example.com/robots.txt",+ b"Sitemap: http://www.example.com/sitemap1.xml",+ [+ (+ "http://www.example.com/sitemap1.xml",+ "_parse_sitemap",+ {"loc": "http://www.example.com/sitemap1.xml"},+ "robotstxt",+ )+ ],+ ),+ (+ "http://www.example.com/sitemap.xml",+ b"""<?xml version="1.0" encoding="UTF-8"?>+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <sitemap>+ <loc>http://www.example.com/sitemap1.xml</loc>+ <lastmod>2005-01-01</lastmod>+ </sitemap>+ </sitemapindex>""",+ [+ (+ "http://www.example.com/sitemap1.xml",+ "_parse_sitemap",+ {+ "loc": "http://www.example.com/sitemap1.xml",+ "lastmod": "2005-01-01",+ },+ "sitemapindex",+ )+ ],+ ),+ (+ "http://www.example.com/sitemap.xml",+ b"""<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">+ <url>+ <loc>http://www.example.com/english/</loc>+ <lastmod>2005-01-01</lastmod>+ </url>+ </urlset>""",+ [+ (+ "http://www.example.com/english/",+ "parse",+ {+ "loc": "http://www.example.com/english/",+ "lastmod": "2005-01-01",+ },+ "urlset",+ )+ ],+ ),+ ],+ ids=["robotstxt", "sitemapindex", "urlset"],+ )+ def test_sitemap_request_source(+ self,+ url: str,+ body: bytes,+ expected: list[tuple[str, str, dict[str, str], str]],+ ):+ response = TextResponse(url=url, body=body)+ calls = []++ class _Spider(self.spider_class): # type: ignore[name-defined,misc]+ def sitemap_request(self, loc, callback, entry, *, source, response):+ calls.append((loc, callback.__name__, entry, source, response))+ return super().sitemap_request(+ loc, callback, entry, source=source, response=response+ )++ spider = _Spider("example.com")+ requests = list(spider._parse_sitemap(response))++ assert [request.url for request in requests] == [loc for loc, *_ in expected]+ assert calls == [(*item, response) for item in expected]++ def test_sitemap_request_for_alternate_links(self):+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"+ xmlns:xhtml="http://www.w3.org/1999/xhtml">+ <url>+ <loc>http://www.example.com/english/</loc>+ <xhtml:link rel="alternate" hreflang="de" href="http://www.example.com/deutsch/"/>+ </url>+ </urlset>"""+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)++ class _Spider(self.spider_class): # type: ignore[name-defined,misc]+ sitemap_alternate_links = True++ def sitemap_request(self, loc, callback, entry, *, source, response):+ return Request(loc, callback, meta={"entry_loc": entry["loc"]})++ spider = _Spider("example.com")+ requests = list(spider._parse_sitemap(r))++ assert [request.meta["entry_loc"] for request in requests] == [+ "http://www.example.com/english/",+ "http://www.example.com/english/",+ ]+def test_compression_bomb_setting(self):settings = {"DOWNLOAD_MAXSIZE": 10_000_000}crawler = get_crawler(settings_dict=settings)@@ -706,3 +895,12 @@ Sitemap: /sitemap-relative-url.xmlassert request.url == "https://toscrape.com/sitemap.xml"assert request.dont_filter is Falseassert request.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable+++def test_iterloc_deprecated():+ entries = [{"loc": "http://www.example.com/"}]++ with pytest.warns(ScrapyDeprecationWarning, match="iterloc"):+ locs = list(iterloc(entries))++ assert locs == ["http://www.example.com/"]