Scrapy, a fast high-level web crawling & scraping framework for Python.
SitemapSpider: avoid duplicate robots.txt requests (#8069)
Adrianb2fc90bef92c
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 454958868..3d4a537fd 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -111,9 +111,53 @@ class RobotsTxtMiddleware:returnif request.url.startswith("data:") or request.url.startswith("file:"):return+ netloc = urlparse_cached(request).netloc+ if netloc not in self._parsers and self._is_robotstxt_request(request):+ # Let this request download normally and use its own response as+ # this host's robots.txt, instead of triggering a second,+ # separate download of the same URL.+ self._parsers[netloc] = Deferred()+ self._stats.inc_value("robotstxt/request_count")+ returnrp = await self.robot_parser(request)self.process_request_2(rp, request)+ @_warn_spider_arg+ async def process_response(+ self, request: Request, response: Response, spider: Spider | None = None+ ) -> Response:+ netloc = urlparse_cached(request).netloc+ parser = self._parsers.get(netloc)+ if (+ isinstance(parser, Deferred)+ and not parser.called+ and self._is_robotstxt_request(request)+ ):+ await self._parse_robots(response, netloc, request)+ return response++ @_warn_spider_arg+ async def process_exception(+ self, request: Request, exception: Exception, spider: Spider | None = None+ ) -> None:+ netloc = urlparse_cached(request).netloc+ parser = self._parsers.get(netloc)+ if (+ isinstance(parser, Deferred)+ and not parser.called+ and self._is_robotstxt_request(request)+ ):+ self._robots_error(exception, netloc, request)++ @staticmethod+ def _is_robotstxt_request(request: Request) -> bool:+ # robot_parser() handles its own internal robots.txt request directly,+ # without going through process_response()/process_exception().+ if request.meta.get("dont_obey_robotstxt"):+ return False+ url = urlparse_cached(request)+ return request.url == f"{url.scheme}://{url.netloc}/robots.txt"+def process_request_2(self, rp: RobotParser | None, request: Request) -> None:if rp is None:return@@ -156,12 +200,7 @@ class RobotsTxtMiddleware:resp = await self.crawler.engine.download_async(robotsreq)await self._parse_robots(resp, netloc, request)except Exception as e:- if not isinstance(e, IgnoreRequest):- logger.exception(- f"Error downloading {request}",- extra={"spider": self.crawler.spider},- )- self._robots_error(e, netloc)+ self._robots_error(e, netloc, request)self._stats.inc_value("robotstxt/request_count")parser = self._parsers[netloc]@@ -185,8 +224,13 @@ class RobotsTxtMiddleware:self._parsers[netloc] = rprp_dfd.callback(rp)- def _robots_error(self, exc: Exception, netloc: str) -> None:+ def _robots_error(self, exc: Exception, netloc: str, request: Request) -> None:if not isinstance(exc, IgnoreRequest):+ logger.error(+ f"Error downloading {request}",+ exc_info=exc,+ extra={"spider": self.crawler.spider},+ )key = f"robotstxt/exception_count/{type(exc)}"self._stats.inc_value(key)rp_dfd = self._parsers[netloc]diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.pyindex ba8740978..9313a59e3 100644--- a/scrapy/robotstxt.py+++ b/scrapy/robotstxt.py@@ -11,6 +11,8 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import to_unicodeif TYPE_CHECKING:+ from collections.abc import Iterable+from scrapy import Spiderfrom scrapy.crawler import Crawler@@ -71,6 +73,15 @@ class RobotParser(metaclass=ABCMeta):"""return None+ def sitemaps(self) -> Iterable[str]:+ """Return the sitemap URLs declared through the ``Sitemap`` directive+ of the robots.txt file, or an empty iterable if there are none or the+ backend does not support reading them.++ .. versionadded:: VERSION+ """+ return ()+class PythonRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):@@ -117,6 +128,9 @@ class RerpRobotParser(RobotParser):delay = self.rp.get_crawl_delay(to_unicode(user_agent))return None if delay is None else float(delay)+ def sitemaps(self) -> Iterable[str]:+ return cast("Iterable[str]", self.rp.sitemaps)+class ProtegoRobotParser(RobotParser):def __init__(self, robotstxt_body: bytes, spider: Spider | None):@@ -137,3 +151,6 @@ class ProtegoRobotParser(RobotParser):def crawl_delay(self, user_agent: str | bytes) -> float | None:delay = self.rp.crawl_delay(to_unicode(user_agent))return None if delay is None else float(delay)++ def sitemaps(self) -> Iterable[str]:+ return self.rp.sitemapsdiff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.pyindex 0c20c1e1d..0c14a4cc6 100644--- a/scrapy/spiders/sitemap.py+++ b/scrapy/spiders/sitemap.py@@ -8,6 +8,7 @@ from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003from typing import TYPE_CHECKING, Any, Self, castfrom urllib.parse import urljoin+from scrapy import signalsfrom scrapy.http import Request, Response, XmlResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.spiders import Spider@@ -17,10 +18,12 @@ from scrapy.utils._compression import (gzip_magic_number,)from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots+from scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from scrapy.crawler import Crawlerfrom scrapy.http.request import CallbackT+ from scrapy.robotstxt import RobotParserlogger = logging.getLogger(__name__)@@ -93,6 +96,7 @@ class SitemapSpider(Spider):_max_size: int_warn_size: int+ _obey_robotstxt: bool@classmethoddef from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:@@ -103,6 +107,9 @@ class SitemapSpider(Spider):spider._warn_size = getattr(spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE"))+ spider._obey_robotstxt = crawler.settings.getbool("ROBOTSTXT_OBEY")+ if spider._obey_robotstxt:+ crawler.signals.connect(spider._robots_parsed, signals.robots_parsed)return spiderdef __init__(self, *a: Any, **kw: Any):@@ -113,9 +120,41 @@ class SitemapSpider(Spider):c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901self._cbs.append((regex(r), c))self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]+ self._robots_sitemaps: dict[str, list[str]] = {}+ self._signal_only_netlocs: set[str] = set()++ def _robots_parsed(self, robotparser: RobotParser, request: Request) -> None:+ netloc = urlparse_cached(request).netloc+ urls = [urljoin(request.url, url) for url in robotparser.sitemaps()]+ if netloc in self._signal_only_netlocs:+ # No request for this host's robots.txt was sent from start(), so+ # _parse_sitemap() will never run for it; schedule its sitemaps+ # directly instead of stashing them for a response that never arrives.+ self._signal_only_netlocs.discard(netloc)+ for url in urls:+ self.crawler.engine.crawl(Request(url, callback=self._parse_sitemap))+ else:+ self._robots_sitemaps[netloc] = urlsasync def start(self) -> AsyncIterator[Any]:+ sitemap_netlocs = {+ urlparse_cached(url).netloc+ for url in self.sitemap_urls+ if not url.endswith("/robots.txt")+ }for url in self.sitemap_urls:+ netloc = urlparse_cached(url).netloc+ if (+ self._obey_robotstxt+ and url.endswith("/robots.txt")+ and netloc in sitemap_netlocs+ ):+ # RobotsTxtMiddleware will fetch and parse this robots.txt+ # anyway, to check the other sitemap_urls requests to the same+ # host, so rely on the robots_parsed signal for it instead of+ # requesting it a second time here.+ self._signal_only_netlocs.add(netloc)+ continueyield Request(url, self._parse_sitemap)def sitemap_filter(@@ -175,6 +214,10 @@ class SitemapSpider(Spider):def _parse_sitemap(self, response: Response) -> Iterable[Request]:if response.url.endswith("/robots.txt"):urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))+ netloc = urlparse_cached(response).netloc+ for url in self._robots_sitemaps.pop(netloc, ()):+ if url not in urls:+ urls.append(url)return (Request(url, callback=self._parse_sitemap) for url in urls)body = self._get_sitemap_body(response)diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex f385b73e4..c7e11ccef 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -222,6 +222,36 @@ Disallow: /some/randome/page.htmlmiddleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)await self.assertNotIgnored(Request("http://site.local"), middleware)+ @coroutine_test+ async def test_robotstxt_request_download_error(+ self, caplog: pytest.LogCaptureFixture+ ) -> None:+ self.crawler.settings.set("ROBOTSTXT_OBEY", True)+ middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)+ request = Request("http://site.local/robots.txt")+ await middleware.process_request(request)+ err = CannotResolveHostError("Robotstxt address not found")+ await middleware.process_exception(request, err)+ assert "Robotstxt address not found" in caplog.text+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)++ @coroutine_test+ async def test_robotstxt_process_exception_ignores_unrelated_request(+ self,+ ) -> None:+ middleware = build_from_crawler(+ RobotsTxtMiddleware, self._get_successful_crawler()+ )+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)+ err = CannotResolveHostError("Unrelated request failed")+ with mock.patch(+ "scrapy.downloadermiddlewares.robotstxt.logger"+ ) as mw_module_logger:+ await middleware.process_exception(+ Request("http://site.local/allowed"), err+ )+ assert not mw_module_logger.error.called+@coroutine_testasync def test_ignore_robotstxt_request(self):self.crawler.settings.set("ROBOTSTXT_OBEY", True)diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.pyindex bfb249f2b..35619a839 100644--- a/tests/test_robotstxt_interface.py+++ b/tests/test_robotstxt_interface.py@@ -106,6 +106,19 @@ class TestRobotParserBase(ABC):rp = self._parse(robotstxt_body)assert rp.crawl_delay("*") is None+ def test_sitemaps(self) -> None:+ robotstxt_body = (+ b"User-agent: *\nDisallow: /private\n"+ b"Sitemap: https://site.local/sitemap.xml\n"+ )+ rp = self._parse(robotstxt_body)+ assert list(rp.sitemaps()) == ["https://site.local/sitemap.xml"]++ def test_sitemaps_unset(self) -> None:+ robotstxt_body = b"User-agent: *\nDisallow: /private\n"+ rp = self._parse(robotstxt_body)+ assert list(rp.sitemaps()) == []+def test_unicode_url_and_useragent(self) -> None:robotstxt_robotstxt_body = """User-Agent: *@@ -143,6 +156,22 @@ class TestRobotParser:)assert rp.crawl_delay("*") is None+ def test_sitemaps_unsupported(self) -> None:+ class AllowAllRobotParser(RobotParser):+ @classmethod+ def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:+ return cls()++ def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:+ return True++ rp = build_from_crawler(+ AllowAllRobotParser,+ NO_CRAWLER,+ b"User-agent: *\nSitemap: https://site.local/sitemap.xml\n",+ )+ assert list(rp.sitemaps()) == []+class TestDecodeRobotsTxt:def test_decode_utf8(self) -> None:@@ -186,6 +215,9 @@ class TestPythonRobotParser(TestRobotParserBase):def test_allowed_wildcards(self) -> None:super().test_allowed_wildcards()+ def test_sitemaps(self) -> None:+ pytest.skip("RobotFileParser does not support the Sitemap directive.")+@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")class TestRerpRobotParser(TestRobotParserBase):diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.pyindex 6f50e4547..da5b2b24d 100644--- a/tests/test_spider_sitemap.py+++ b/tests/test_spider_sitemap.py@@ -7,9 +7,11 @@ from io import BytesIOfrom logging import WARNINGfrom pathlib import Pathfrom typing import TYPE_CHECKING+from unittest import mockimport pytest+from scrapy import signalsfrom scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponsefrom scrapy.spiders import SitemapSpiderfrom scrapy.utils.test import get_crawler@@ -147,6 +149,67 @@ Sitemap: /sitemap-relative-url.xml"http://example.com/ok.xml",]+ def test_get_sitemap_urls_from_robots_parsed_signal(self):+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})+ spider = self.spider_class.from_crawler(crawler, "example.com")++ class FakeRobotParser:+ def sitemaps(self):+ return [+ "http://example.com/sitemap.xml", # already found below+ "/sitemap-from-parser.xml", # only reported through the signal+ ]++ crawler.signals.send_catch_log(+ signal=signals.robots_parsed,+ robotparser=FakeRobotParser(),+ request=Request("http://www.example.com/robots.txt"),+ )++ robots = b"Sitemap: http://example.com/sitemap.xml\n"+ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)+ assert [req.url for req in spider._parse_sitemap(r)] == [+ "http://example.com/sitemap.xml",+ "http://www.example.com/sitemap-from-parser.xml",+ ]++ def test_robots_parsed_signal_not_connected_without_robotstxt_obey(self):+ crawler = get_crawler(self.spider_class)+ spider = self.spider_class.from_crawler(crawler, "example.com")++ crawler.signals.send_catch_log(+ signal=signals.robots_parsed,+ robotparser=None,+ request=Request("http://www.example.com/robots.txt"),+ )++ assert spider._robots_sitemaps == {}++ def test_robots_parsed_signal_schedules_sitemaps_for_skipped_netloc(self):+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})+ spider = self.spider_class.from_crawler(crawler, "example.com")+ crawler.engine = mock.MagicMock()+ spider._signal_only_netlocs.add("www.example.com")++ class FakeRobotParser:+ def sitemaps(self):+ return ["/sitemap.xml"]++ # request is whatever other request to that host triggered the fetch,+ # not necessarily a request for robots.txt itself.+ crawler.signals.send_catch_log(+ signal=signals.robots_parsed,+ robotparser=FakeRobotParser(),+ request=Request("http://www.example.com/some/page"),+ )++ crawler.engine.crawl.assert_called_once()+ scheduled = crawler.engine.crawl.call_args[0][0]+ assert scheduled.url == "http://www.example.com/sitemap.xml"+ assert scheduled.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable+ assert spider._signal_only_netlocs == set()+ assert spider._robots_sitemaps == {}+def test_alternate_url_locs(self):sitemap = b"""<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"@@ -356,6 +419,93 @@ Sitemap: /sitemap-relative-url.xmlitems, _ = await crawl_items(_Spider, mockserver)assert items == [{"url": mockserver.url("/text")}]+ @coroutine_test+ async def test_robots_txt_requested_once(self, mockserver: MockServer):+ """robots.txt should be downloaded only once, even though both+ SitemapSpider and RobotsTxtMiddleware need its content: the spider+ requests it directly to look for sitemaps, while the middleware+ downloads it to know what the spider is allowed to crawl.+ """+ robots_url = mockserver.url("/robots.txt")++ class _Spider(self.spider_class): # type: ignore[name-defined,misc]+ name = "robots_txt_requested_once"+ custom_settings = {"ROBOTSTXT_OBEY": True}+ sitemap_urls = [robots_url]++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.robots_txt_requests = []+ crawler.signals.connect(+ spider._track_downloader_hit,+ signals.request_reached_downloader,+ )+ return spider++ def _track_downloader_hit(self, request, spider):+ if request.url == robots_url:+ self.robots_txt_requests.append(request)++ _, crawler = await crawl_items(_Spider, mockserver)+ assert isinstance(crawler.spider, _Spider)+ assert len(crawler.spider.robots_txt_requests) == 1++ @coroutine_test+ async def test_start_skips_robots_url_covered_by_another_sitemap_url(self):+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})+ spider = self.spider_class.from_crawler(+ crawler,+ "example.com",+ sitemap_urls=[+ "http://www.example.com/sitemap.xml",+ "http://www.example.com/robots.txt",+ "http://other.example.com/robots.txt",+ ],+ )++ urls = [request.url async for request in spider.start()]++ assert urls == [+ "http://www.example.com/sitemap.xml",+ "http://other.example.com/robots.txt",+ ]+ assert spider._signal_only_netlocs == {"www.example.com"}++ @coroutine_test+ async def test_robots_txt_requested_once_when_other_sitemap_url_goes_first(+ self, mockserver: MockServer+ ):+ """When another sitemap_urls entry targets the same host, the spider+ should not also request robots.txt itself: RobotsTxtMiddleware's own+ robots.txt request, triggered by that other entry, covers it.+ """+ robots_url = mockserver.url("/robots.txt")+ other_url = mockserver.url("/text")++ class _Spider(self.spider_class): # type: ignore[name-defined,misc]+ name = "robots_txt_requested_once_other_first"+ custom_settings = {"ROBOTSTXT_OBEY": True}+ sitemap_urls = [other_url, robots_url]++ @classmethod+ def from_crawler(cls, crawler, *args, **kwargs):+ spider = super().from_crawler(crawler, *args, **kwargs)+ spider.robots_txt_requests = []+ crawler.signals.connect(+ spider._track_downloader_hit,+ signals.request_reached_downloader,+ )+ return spider++ def _track_downloader_hit(self, request, spider):+ if request.url == robots_url:+ self.robots_txt_requests.append(request)++ _, crawler = await crawl_items(_Spider, mockserver)+ assert isinstance(crawler.spider, _Spider)+ assert len(crawler.spider.robots_txt_requests) == 1+def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")spider = self.spider_class("example.com")