Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add SitemapSpider.sitemap_request() to build requests from sitemap entries (#8175)

Adrianc38f20cf3b53
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rst
index e05a4518d..3164dbe75 100644
--- a/docs/topics/signals.rst
+++ b/docs/topics/signals.rst
@@ -607,10 +607,13 @@ robots_parsed
~~~~~~~~~~~~~
.. signal:: robots_parsed
-.. function:: robots_parsed(robotparser, request)
+.. function:: robots_parsed(robotparser, request, response)
.. versionadded:: 2.18.0
+ .. versionchanged:: VERSION
+ Added *response*.
+
Sent by
:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` after it
downloads and parses a :file:`robots.txt` file, for the host that *request*
@@ -624,6 +627,9 @@ robots_parsed
:param request: the request that triggered the :file:`robots.txt` download
:type request: :class:`~scrapy.Request` object
+ :param response: the :file:`robots.txt` response
+ :type response: :class:`~scrapy.http.Response` object
+
Response signals
----------------
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 3d4a537fd..a0525af79 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -218,6 +218,7 @@ class RobotsTxtMiddleware:
signal=signals.robots_parsed,
robotparser=rp,
request=request,
+ response=response,
)
rp_dfd = self._parsers[netloc]
assert isinstance(rp_dfd, Deferred)
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 0c14a4cc6..a54324b00 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -2,13 +2,15 @@ from __future__ import annotations
import logging
import re
+import warnings
# Iterable is needed at the run time for the SitemapSpider._parse_sitemap() annotation
from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003
-from typing import TYPE_CHECKING, Any, Self, cast
+from typing import TYPE_CHECKING, Any, Literal, Self, cast
from urllib.parse import urljoin
from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response, XmlResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -69,9 +71,10 @@ class SitemapSpider(Spider):
"""
sitemap_follow: Sequence[re.Pattern[str] | str] = [""]
- """Regexes of sitemaps that should be followed. This is only for sites that
- use `sitemap index files <https://www.sitemaps.org/protocol.html#index>`_
- that point to other sitemap files.
+ """Regexes of sitemap URLs that should be followed: those linked from
+ `sitemap index files <https://www.sitemaps.org/protocol.html#index>`_, and
+ those declared through the ``Sitemap`` directive of a :file:`robots.txt`
+ file.
By default, all sitemaps are followed.
"""
@@ -123,7 +126,9 @@ class SitemapSpider(Spider):
self._robots_sitemaps: dict[str, list[str]] = {}
self._signal_only_netlocs: set[str] = set()
- def _robots_parsed(self, robotparser: RobotParser, request: Request) -> None:
+ def _robots_parsed(
+ self, robotparser: RobotParser, request: Request, response: Response
+ ) -> None:
netloc = urlparse_cached(request).netloc
urls = [urljoin(request.url, url) for url in robotparser.sitemaps()]
if netloc in self._signal_only_netlocs:
@@ -132,7 +137,16 @@ class SitemapSpider(Spider):
# directly instead of stashing them for a response that never arrives.
self._signal_only_netlocs.discard(netloc)
for url in urls:
- self.crawler.engine.crawl(Request(url, callback=self._parse_sitemap))
+ if any(x.search(url) for x in self._follow):
+ self.crawler.engine.crawl(
+ self.sitemap_request(
+ url,
+ self._parse_sitemap,
+ {"loc": url},
+ source="robotstxt",
+ response=response,
+ )
+ )
else:
self._robots_sitemaps[netloc] = urls
@@ -211,6 +225,45 @@ class SitemapSpider(Spider):
"""
yield from entries
+ def sitemap_request(
+ self,
+ loc: str,
+ callback: CallbackT,
+ entry: dict[str, Any],
+ *,
+ source: Literal["robotstxt", "sitemapindex", "urlset"],
+ response: Response,
+ ) -> Request:
+ """Return the request for the *loc* URL of the sitemap *entry*, to be
+ parsed with *callback*.
+
+ .. versionadded:: VERSION
+
+ Override this method to build those requests differently, e.g. to
+ carry sitemap data and request metadata over to *callback*:
+
+ .. code-block:: python
+
+ def sitemap_request(self, loc, callback, entry, *, source, response):
+ meta = {
+ "lastmod": entry.get("lastmod"),
+ "site": response.meta.get("site"),
+ }
+ return Request(loc, callback, meta=meta)
+
+ *entry* is the sitemap entry that *loc* comes from, as described in
+ :meth:`sitemap_filter`. When :attr:`sitemap_alternate_links` is
+ enabled, the same entry is used for every alternate URL.
+
+ *source* tells where *loc* comes from: ``"urlset"`` for a URL to
+ crawl, ``"sitemapindex"`` for a sitemap linked from a sitemap index,
+ and ``"robotstxt"`` for a sitemap linked from a :file:`robots.txt`
+ file, where *entry* only has ``loc``.
+
+ *response* is the response that *entry* was read from.
+ """
+ return Request(loc, callback=callback)
+
def _parse_sitemap(self, response: Response) -> Iterable[Request]:
if response.url.endswith("/robots.txt"):
urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))
@@ -218,7 +271,17 @@ class SitemapSpider(Spider):
for url in self._robots_sitemaps.pop(netloc, ()):
if url not in urls:
urls.append(url)
- return (Request(url, callback=self._parse_sitemap) for url in urls)
+ return (
+ self.sitemap_request(
+ url,
+ self._parse_sitemap,
+ {"loc": url},
+ source="robotstxt",
+ response=response,
+ )
+ for url in urls
+ if any(x.search(url) for x in self._follow)
+ )
body = self._get_sitemap_body(response)
if not body:
@@ -232,18 +295,30 @@ class SitemapSpider(Spider):
s = Sitemap(body)
if s.type == "sitemapindex":
- urls = list(
+ index_entries = list(
self._get_urls_from_sitemapindex(self.sitemap_filter(s), response.url)
)
- return (Request(loc, callback=self._parse_sitemap) for loc in urls)
+ return (
+ self.sitemap_request(
+ loc,
+ self._parse_sitemap,
+ entry,
+ source="sitemapindex",
+ response=response,
+ )
+ for loc, entry in index_entries
+ )
if s.type == "urlset":
- url_callback_pairs = list(
+ urlset_entries = list(
self._get_urls_and_callbacks_from_urlset(
self.sitemap_filter(s), response.url
)
)
- return (Request(loc, callback=c) for loc, c in url_callback_pairs)
+ return (
+ self.sitemap_request(loc, c, entry, source="urlset", response=response)
+ for loc, c, entry in urlset_entries
+ )
logger.warning(
"Ignoring invalid sitemap: %(response)s",
@@ -255,20 +330,20 @@ class SitemapSpider(Spider):
def _get_urls_from_sitemapindex(
self, it: Iterable[dict[str, Any]], base_url: str
- ) -> Iterable[str]:
- for loc in iterloc(it, self.sitemap_alternate_links):
+ ) -> Iterable[tuple[str, dict[str, Any]]]:
+ for loc, entry in _iterlocs(it, self.sitemap_alternate_links):
loc = urljoin(base_url, loc) # noqa: PLW2901
if any(x.search(loc) for x in self._follow):
- yield loc
+ yield loc, entry
def _get_urls_and_callbacks_from_urlset(
self, it: Iterable[dict[str, Any]], base_url: str
- ) -> Iterable[tuple[str, CallbackT]]:
- for loc in iterloc(it, self.sitemap_alternate_links):
+ ) -> Iterable[tuple[str, CallbackT, dict[str, Any]]]:
+ for loc, entry in _iterlocs(it, self.sitemap_alternate_links):
loc = urljoin(base_url, loc) # noqa: PLW2901
for r, c in self._cbs:
if r.search(loc):
- yield loc, c
+ yield loc, c, entry
break
def _get_sitemap_body(self, response: Response) -> bytes | None:
@@ -324,10 +399,22 @@ def regex(x: re.Pattern[str] | str) -> re.Pattern[str]:
def iterloc(it: Iterable[dict[str, Any]], alt: bool = False) -> Iterable[str]:
+ warnings.warn(
+ "scrapy.spiders.sitemap.iterloc() is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return (loc for loc, _ in _iterlocs(it, alt))
+
+
+def _iterlocs(
+ it: Iterable[dict[str, Any]], alt: bool = False
+) -> Iterable[tuple[str, dict[str, Any]]]:
for d in it:
if loc := d["loc"]:
- yield loc
+ yield loc, d
# Also consider alternate URLs (xhtml:link rel="alternate")
if alt and (alt_list := d.get("alternate")):
- yield from alt_list
+ for loc in alt_list:
+ yield loc, d
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index c7e11ccef..ab809b387 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -96,6 +96,7 @@ Disallow: /some/randome/page.html
assert len(calls) == 1
assert calls[0]["request"] is request
assert calls[0]["robotparser"] is not None
+ assert calls[0]["response"].url == "http://site.local/robots.txt"
@coroutine_test
async def test_robotstxt_multiple_reqs(self) -> None:
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index da5b2b24d..ca5eb0985 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -12,8 +12,10 @@ from unittest import mock
import pytest
from scrapy import signals
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
from scrapy.spiders import SitemapSpider
+from scrapy.spiders.sitemap import iterloc
from scrapy.utils.test import get_crawler
from tests import tests_datadir
from tests.spiders import RawResponseSpider
@@ -164,6 +166,7 @@ Sitemap: /sitemap-relative-url.xml
signal=signals.robots_parsed,
robotparser=FakeRobotParser(),
request=Request("http://www.example.com/robots.txt"),
+ response=TextResponse("http://www.example.com/robots.txt"),
)
robots = b"Sitemap: http://example.com/sitemap.xml\n"
@@ -181,13 +184,23 @@ Sitemap: /sitemap-relative-url.xml
signal=signals.robots_parsed,
robotparser=None,
request=Request("http://www.example.com/robots.txt"),
+ response=TextResponse("http://www.example.com/robots.txt"),
)
assert spider._robots_sitemaps == {}
def test_robots_parsed_signal_schedules_sitemaps_for_skipped_netloc(self):
- crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})
- spider = self.spider_class.from_crawler(crawler, "example.com")
+ calls = []
+
+ class _Spider(self.spider_class): # type: ignore[name-defined,misc]
+ def sitemap_request(self, loc, callback, entry, *, source, response):
+ calls.append((loc, entry, source, response))
+ return super().sitemap_request(
+ loc, callback, entry, source=source, response=response
+ )
+
+ crawler = get_crawler(_Spider, settings_dict={"ROBOTSTXT_OBEY": True})
+ spider = _Spider.from_crawler(crawler, "example.com")
crawler.engine = mock.MagicMock()
spider._signal_only_netlocs.add("www.example.com")
@@ -195,21 +208,49 @@ Sitemap: /sitemap-relative-url.xml
def sitemaps(self):
return ["/sitemap.xml"]
+ robots_response = TextResponse("http://www.example.com/robots.txt")
# request is whatever other request to that host triggered the fetch,
# not necessarily a request for robots.txt itself.
crawler.signals.send_catch_log(
signal=signals.robots_parsed,
robotparser=FakeRobotParser(),
request=Request("http://www.example.com/some/page"),
+ response=robots_response,
)
crawler.engine.crawl.assert_called_once()
scheduled = crawler.engine.crawl.call_args[0][0]
assert scheduled.url == "http://www.example.com/sitemap.xml"
assert scheduled.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable
+ url = "http://www.example.com/sitemap.xml"
+ assert calls == [(url, {"loc": url}, "robotstxt", robots_response)]
assert spider._signal_only_netlocs == set()
assert spider._robots_sitemaps == {}
+ def test_sitemap_follow_robots_parsed_signal(self):
+ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]
+ sitemap_follow = [r"1\.xml"]
+
+ crawler = get_crawler(_FollowSpider, settings_dict={"ROBOTSTXT_OBEY": True})
+ spider = _FollowSpider.from_crawler(crawler, "example.com")
+ crawler.engine = mock.MagicMock()
+ spider._signal_only_netlocs.add("www.example.com")
+
+ class FakeRobotParser:
+ def sitemaps(self):
+ return ["/sitemap1.xml", "/sitemap2.xml"]
+
+ crawler.signals.send_catch_log(
+ signal=signals.robots_parsed,
+ robotparser=FakeRobotParser(),
+ request=Request("http://www.example.com/some/page"),
+ response=TextResponse("http://www.example.com/robots.txt"),
+ )
+
+ crawler.engine.crawl.assert_called_once()
+ scheduled = crawler.engine.crawl.call_args[0][0]
+ assert scheduled.url == "http://www.example.com/sitemap1.xml"
+
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
@@ -561,6 +602,154 @@ Sitemap: /sitemap-relative-url.xml
urls = [req.url for req in spider._parse_sitemap(r)]
assert urls == result
+ @pytest.mark.parametrize(
+ ("follow", "result"),
+ [
+ (r"1.xml", ["http://www.example.com/sitemap1.xml"]),
+ (re.compile(r"sitemap\d"), ["http://www.example.com/sitemap1.xml"]),
+ (r"nonexistent", []),
+ ],
+ )
+ def test_sitemap_follow_robotstxt(self, follow, result):
+ robots = b"Sitemap: http://www.example.com/sitemap1.xml\n"
+ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
+
+ class _FollowSpider(self.spider_class): # type: ignore[name-defined,misc]
+ sitemap_follow = [follow]
+
+ spider = _FollowSpider("example.com")
+ urls = [req.url for req in spider._parse_sitemap(r)]
+ assert urls == result
+
+ @coroutine_test
+ async def test_sitemap_request(self, mockserver: MockServer):
+ class _Spider(RawSitemapSpider, self.spider_class): # type: ignore[name-defined,misc]
+ def sitemap_request(self, loc, callback, entry, *, source, response):
+ return Request(loc, callback, meta={"lastmod": entry["lastmod"]})
+
+ def parse(self, response):
+ yield {"url": response.url, "lastmod": response.meta["lastmod"]}
+
+ def raw_body(self):
+ assert self.mockserver
+ loc = self.mockserver.url("/text")
+ return (
+ '<?xml version="1.0" encoding="UTF-8"?>'
+ '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'
+ f"<url><loc>{loc}</loc><lastmod>2005-01-01</lastmod></url>"
+ "</urlset>"
+ )
+
+ items, _ = await crawl_items(_Spider, mockserver)
+ assert items == [{"url": mockserver.url("/text"), "lastmod": "2005-01-01"}]
+
+ @pytest.mark.parametrize(
+ ("url", "body", "expected"),
+ [
+ (
+ "http://www.example.com/robots.txt",
+ b"Sitemap: http://www.example.com/sitemap1.xml",
+ [
+ (
+ "http://www.example.com/sitemap1.xml",
+ "_parse_sitemap",
+ {"loc": "http://www.example.com/sitemap1.xml"},
+ "robotstxt",
+ )
+ ],
+ ),
+ (
+ "http://www.example.com/sitemap.xml",
+ b"""<?xml version="1.0" encoding="UTF-8"?>
+ <sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <sitemap>
+ <loc>http://www.example.com/sitemap1.xml</loc>
+ <lastmod>2005-01-01</lastmod>
+ </sitemap>
+ </sitemapindex>""",
+ [
+ (
+ "http://www.example.com/sitemap1.xml",
+ "_parse_sitemap",
+ {
+ "loc": "http://www.example.com/sitemap1.xml",
+ "lastmod": "2005-01-01",
+ },
+ "sitemapindex",
+ )
+ ],
+ ),
+ (
+ "http://www.example.com/sitemap.xml",
+ b"""<?xml version="1.0" encoding="UTF-8"?>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+ <url>
+ <loc>http://www.example.com/english/</loc>
+ <lastmod>2005-01-01</lastmod>
+ </url>
+ </urlset>""",
+ [
+ (
+ "http://www.example.com/english/",
+ "parse",
+ {
+ "loc": "http://www.example.com/english/",
+ "lastmod": "2005-01-01",
+ },
+ "urlset",
+ )
+ ],
+ ),
+ ],
+ ids=["robotstxt", "sitemapindex", "urlset"],
+ )
+ def test_sitemap_request_source(
+ self,
+ url: str,
+ body: bytes,
+ expected: list[tuple[str, str, dict[str, str], str]],
+ ):
+ response = TextResponse(url=url, body=body)
+ calls = []
+
+ class _Spider(self.spider_class): # type: ignore[name-defined,misc]
+ def sitemap_request(self, loc, callback, entry, *, source, response):
+ calls.append((loc, callback.__name__, entry, source, response))
+ return super().sitemap_request(
+ loc, callback, entry, source=source, response=response
+ )
+
+ spider = _Spider("example.com")
+ requests = list(spider._parse_sitemap(response))
+
+ assert [request.url for request in requests] == [loc for loc, *_ in expected]
+ assert calls == [(*item, response) for item in expected]
+
+ def test_sitemap_request_for_alternate_links(self):
+ sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
+ <urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
+ xmlns:xhtml="http://www.w3.org/1999/xhtml">
+ <url>
+ <loc>http://www.example.com/english/</loc>
+ <xhtml:link rel="alternate" hreflang="de" href="http://www.example.com/deutsch/"/>
+ </url>
+ </urlset>"""
+ r = TextResponse(url="http://www.example.com/sitemap.xml", body=sitemap)
+
+ class _Spider(self.spider_class): # type: ignore[name-defined,misc]
+ sitemap_alternate_links = True
+
+ def sitemap_request(self, loc, callback, entry, *, source, response):
+ return Request(loc, callback, meta={"entry_loc": entry["loc"]})
+
+ spider = _Spider("example.com")
+ requests = list(spider._parse_sitemap(r))
+
+ assert [request.meta["entry_loc"] for request in requests] == [
+ "http://www.example.com/english/",
+ "http://www.example.com/english/",
+ ]
+
def test_compression_bomb_setting(self):
settings = {"DOWNLOAD_MAXSIZE": 10_000_000}
crawler = get_crawler(settings_dict=settings)
@@ -706,3 +895,12 @@ Sitemap: /sitemap-relative-url.xml
assert request.url == "https://toscrape.com/sitemap.xml"
assert request.dont_filter is False
assert request.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable
+
+
+def test_iterloc_deprecated():
+ entries = [{"loc": "http://www.example.com/"}]
+
+ with pytest.warns(ScrapyDeprecationWarning, match="iterloc"):
+ locs = list(iterloc(entries))
+
+ assert locs == ["http://www.example.com/"]