Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

SitemapSpider: avoid duplicate robots.txt requests (#8069)

Adrianb2fc90bef92c
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 454958868..3d4a537fd 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -111,9 +111,53 @@ class RobotsTxtMiddleware:
return
if request.url.startswith("data:") or request.url.startswith("file:"):
return
+ netloc = urlparse_cached(request).netloc
+ if netloc not in self._parsers and self._is_robotstxt_request(request):
+ # Let this request download normally and use its own response as
+ # this host's robots.txt, instead of triggering a second,
+ # separate download of the same URL.
+ self._parsers[netloc] = Deferred()
+ self._stats.inc_value("robotstxt/request_count")
+ return
rp = await self.robot_parser(request)
self.process_request_2(rp, request)
+ @_warn_spider_arg
+ async def process_response(
+ self, request: Request, response: Response, spider: Spider | None = None
+ ) -> Response:
+ netloc = urlparse_cached(request).netloc
+ parser = self._parsers.get(netloc)
+ if (
+ isinstance(parser, Deferred)
+ and not parser.called
+ and self._is_robotstxt_request(request)
+ ):
+ await self._parse_robots(response, netloc, request)
+ return response
+
+ @_warn_spider_arg
+ async def process_exception(
+ self, request: Request, exception: Exception, spider: Spider | None = None
+ ) -> None:
+ netloc = urlparse_cached(request).netloc
+ parser = self._parsers.get(netloc)
+ if (
+ isinstance(parser, Deferred)
+ and not parser.called
+ and self._is_robotstxt_request(request)
+ ):
+ self._robots_error(exception, netloc, request)
+
+ @staticmethod
+ def _is_robotstxt_request(request: Request) -> bool:
+ # robot_parser() handles its own internal robots.txt request directly,
+ # without going through process_response()/process_exception().
+ if request.meta.get("dont_obey_robotstxt"):
+ return False
+ url = urlparse_cached(request)
+ return request.url == f"{url.scheme}://{url.netloc}/robots.txt"
+
def process_request_2(self, rp: RobotParser | None, request: Request) -> None:
if rp is None:
return
@@ -156,12 +200,7 @@ class RobotsTxtMiddleware:
resp = await self.crawler.engine.download_async(robotsreq)
await self._parse_robots(resp, netloc, request)
except Exception as e:
- if not isinstance(e, IgnoreRequest):
- logger.exception(
- f"Error downloading {request}",
- extra={"spider": self.crawler.spider},
- )
- self._robots_error(e, netloc)
+ self._robots_error(e, netloc, request)
self._stats.inc_value("robotstxt/request_count")
parser = self._parsers[netloc]
@@ -185,8 +224,13 @@ class RobotsTxtMiddleware:
self._parsers[netloc] = rp
rp_dfd.callback(rp)
- def _robots_error(self, exc: Exception, netloc: str) -> None:
+ def _robots_error(self, exc: Exception, netloc: str, request: Request) -> None:
if not isinstance(exc, IgnoreRequest):
+ logger.error(
+ f"Error downloading {request}",
+ exc_info=exc,
+ extra={"spider": self.crawler.spider},
+ )
key = f"robotstxt/exception_count/{type(exc)}"
self._stats.inc_value(key)
rp_dfd = self._parsers[netloc]
diff --git a/scrapy/robotstxt.py b/scrapy/robotstxt.py
index ba8740978..9313a59e3 100644
--- a/scrapy/robotstxt.py
+++ b/scrapy/robotstxt.py
@@ -11,6 +11,8 @@ from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.python import to_unicode
if TYPE_CHECKING:
+ from collections.abc import Iterable
+
from scrapy import Spider
from scrapy.crawler import Crawler
@@ -71,6 +73,15 @@ class RobotParser(metaclass=ABCMeta):
"""
return None
+ def sitemaps(self) -> Iterable[str]:
+ """Return the sitemap URLs declared through the ``Sitemap`` directive
+ of the robots.txt file, or an empty iterable if there are none or the
+ backend does not support reading them.
+
+ .. versionadded:: VERSION
+ """
+ return ()
+
class PythonRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -117,6 +128,9 @@ class RerpRobotParser(RobotParser):
delay = self.rp.get_crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
+ def sitemaps(self) -> Iterable[str]:
+ return cast("Iterable[str]", self.rp.sitemaps)
+
class ProtegoRobotParser(RobotParser):
def __init__(self, robotstxt_body: bytes, spider: Spider | None):
@@ -137,3 +151,6 @@ class ProtegoRobotParser(RobotParser):
def crawl_delay(self, user_agent: str | bytes) -> float | None:
delay = self.rp.crawl_delay(to_unicode(user_agent))
return None if delay is None else float(delay)
+
+ def sitemaps(self) -> Iterable[str]:
+ return self.rp.sitemaps
diff --git a/scrapy/spiders/sitemap.py b/scrapy/spiders/sitemap.py
index 0c20c1e1d..0c14a4cc6 100644
--- a/scrapy/spiders/sitemap.py
+++ b/scrapy/spiders/sitemap.py
@@ -8,6 +8,7 @@ from collections.abc import AsyncIterator, Iterable, Sequence # noqa: TC003
from typing import TYPE_CHECKING, Any, Self, cast
from urllib.parse import urljoin
+from scrapy import signals
from scrapy.http import Request, Response, XmlResponse
from scrapy.responsetypes import responsetypes
from scrapy.spiders import Spider
@@ -17,10 +18,12 @@ from scrapy.utils._compression import (
gzip_magic_number,
)
from scrapy.utils._sitemap import Sitemap, sitemap_urls_from_robots
+from scrapy.utils.httpobj import urlparse_cached
if TYPE_CHECKING:
from scrapy.crawler import Crawler
from scrapy.http.request import CallbackT
+ from scrapy.robotstxt import RobotParser
logger = logging.getLogger(__name__)
@@ -93,6 +96,7 @@ class SitemapSpider(Spider):
_max_size: int
_warn_size: int
+ _obey_robotstxt: bool
@classmethod
def from_crawler(cls, crawler: Crawler, *args: Any, **kwargs: Any) -> Self:
@@ -103,6 +107,9 @@ class SitemapSpider(Spider):
spider._warn_size = getattr(
spider, "download_warnsize", spider.settings.getint("DOWNLOAD_WARNSIZE")
)
+ spider._obey_robotstxt = crawler.settings.getbool("ROBOTSTXT_OBEY")
+ if spider._obey_robotstxt:
+ crawler.signals.connect(spider._robots_parsed, signals.robots_parsed)
return spider
def __init__(self, *a: Any, **kw: Any):
@@ -113,9 +120,41 @@ class SitemapSpider(Spider):
c = cast("CallbackT", getattr(self, c)) # noqa: PLW2901
self._cbs.append((regex(r), c))
self._follow: list[re.Pattern[str]] = [regex(x) for x in self.sitemap_follow]
+ self._robots_sitemaps: dict[str, list[str]] = {}
+ self._signal_only_netlocs: set[str] = set()
+
+ def _robots_parsed(self, robotparser: RobotParser, request: Request) -> None:
+ netloc = urlparse_cached(request).netloc
+ urls = [urljoin(request.url, url) for url in robotparser.sitemaps()]
+ if netloc in self._signal_only_netlocs:
+ # No request for this host's robots.txt was sent from start(), so
+ # _parse_sitemap() will never run for it; schedule its sitemaps
+ # directly instead of stashing them for a response that never arrives.
+ self._signal_only_netlocs.discard(netloc)
+ for url in urls:
+ self.crawler.engine.crawl(Request(url, callback=self._parse_sitemap))
+ else:
+ self._robots_sitemaps[netloc] = urls
async def start(self) -> AsyncIterator[Any]:
+ sitemap_netlocs = {
+ urlparse_cached(url).netloc
+ for url in self.sitemap_urls
+ if not url.endswith("/robots.txt")
+ }
for url in self.sitemap_urls:
+ netloc = urlparse_cached(url).netloc
+ if (
+ self._obey_robotstxt
+ and url.endswith("/robots.txt")
+ and netloc in sitemap_netlocs
+ ):
+ # RobotsTxtMiddleware will fetch and parse this robots.txt
+ # anyway, to check the other sitemap_urls requests to the same
+ # host, so rely on the robots_parsed signal for it instead of
+ # requesting it a second time here.
+ self._signal_only_netlocs.add(netloc)
+ continue
yield Request(url, self._parse_sitemap)
def sitemap_filter(
@@ -175,6 +214,10 @@ class SitemapSpider(Spider):
def _parse_sitemap(self, response: Response) -> Iterable[Request]:
if response.url.endswith("/robots.txt"):
urls = list(sitemap_urls_from_robots(response.body, base_url=response.url))
+ netloc = urlparse_cached(response).netloc
+ for url in self._robots_sitemaps.pop(netloc, ()):
+ if url not in urls:
+ urls.append(url)
return (Request(url, callback=self._parse_sitemap) for url in urls)
body = self._get_sitemap_body(response)
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f385b73e4..c7e11ccef 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -222,6 +222,36 @@ Disallow: /some/randome/page.html
middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
await self.assertNotIgnored(Request("http://site.local"), middleware)
+ @coroutine_test
+ async def test_robotstxt_request_download_error(
+ self, caplog: pytest.LogCaptureFixture
+ ) -> None:
+ self.crawler.settings.set("ROBOTSTXT_OBEY", True)
+ middleware = build_from_crawler(RobotsTxtMiddleware, self.crawler)
+ request = Request("http://site.local/robots.txt")
+ await middleware.process_request(request)
+ err = CannotResolveHostError("Robotstxt address not found")
+ await middleware.process_exception(request, err)
+ assert "Robotstxt address not found" in caplog.text
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+
+ @coroutine_test
+ async def test_robotstxt_process_exception_ignores_unrelated_request(
+ self,
+ ) -> None:
+ middleware = build_from_crawler(
+ RobotsTxtMiddleware, self._get_successful_crawler()
+ )
+ await self.assertNotIgnored(Request("http://site.local/allowed"), middleware)
+ err = CannotResolveHostError("Unrelated request failed")
+ with mock.patch(
+ "scrapy.downloadermiddlewares.robotstxt.logger"
+ ) as mw_module_logger:
+ await middleware.process_exception(
+ Request("http://site.local/allowed"), err
+ )
+ assert not mw_module_logger.error.called
+
@coroutine_test
async def test_ignore_robotstxt_request(self):
self.crawler.settings.set("ROBOTSTXT_OBEY", True)
diff --git a/tests/test_robotstxt_interface.py b/tests/test_robotstxt_interface.py
index bfb249f2b..35619a839 100644
--- a/tests/test_robotstxt_interface.py
+++ b/tests/test_robotstxt_interface.py
@@ -106,6 +106,19 @@ class TestRobotParserBase(ABC):
rp = self._parse(robotstxt_body)
assert rp.crawl_delay("*") is None
+ def test_sitemaps(self) -> None:
+ robotstxt_body = (
+ b"User-agent: *\nDisallow: /private\n"
+ b"Sitemap: https://site.local/sitemap.xml\n"
+ )
+ rp = self._parse(robotstxt_body)
+ assert list(rp.sitemaps()) == ["https://site.local/sitemap.xml"]
+
+ def test_sitemaps_unset(self) -> None:
+ robotstxt_body = b"User-agent: *\nDisallow: /private\n"
+ rp = self._parse(robotstxt_body)
+ assert list(rp.sitemaps()) == []
+
def test_unicode_url_and_useragent(self) -> None:
robotstxt_robotstxt_body = """
User-Agent: *
@@ -143,6 +156,22 @@ class TestRobotParser:
)
assert rp.crawl_delay("*") is None
+ def test_sitemaps_unsupported(self) -> None:
+ class AllowAllRobotParser(RobotParser):
+ @classmethod
+ def from_crawler(cls, crawler: Crawler, robotstxt_body: bytes) -> Self:
+ return cls()
+
+ def allowed(self, url: str | bytes, user_agent: str | bytes) -> bool:
+ return True
+
+ rp = build_from_crawler(
+ AllowAllRobotParser,
+ NO_CRAWLER,
+ b"User-agent: *\nSitemap: https://site.local/sitemap.xml\n",
+ )
+ assert list(rp.sitemaps()) == []
+
class TestDecodeRobotsTxt:
def test_decode_utf8(self) -> None:
@@ -186,6 +215,9 @@ class TestPythonRobotParser(TestRobotParserBase):
def test_allowed_wildcards(self) -> None:
super().test_allowed_wildcards()
+ def test_sitemaps(self) -> None:
+ pytest.skip("RobotFileParser does not support the Sitemap directive.")
+
@pytest.mark.skipif(not rerp_available(), reason="Rerp parser is not installed")
class TestRerpRobotParser(TestRobotParserBase):
diff --git a/tests/test_spider_sitemap.py b/tests/test_spider_sitemap.py
index 6f50e4547..da5b2b24d 100644
--- a/tests/test_spider_sitemap.py
+++ b/tests/test_spider_sitemap.py
@@ -7,9 +7,11 @@ from io import BytesIO
from logging import WARNING
from pathlib import Path
from typing import TYPE_CHECKING
+from unittest import mock
import pytest
+from scrapy import signals
from scrapy.http import HtmlResponse, Request, Response, TextResponse, XmlResponse
from scrapy.spiders import SitemapSpider
from scrapy.utils.test import get_crawler
@@ -147,6 +149,67 @@ Sitemap: /sitemap-relative-url.xml
"http://example.com/ok.xml",
]
+ def test_get_sitemap_urls_from_robots_parsed_signal(self):
+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+
+ class FakeRobotParser:
+ def sitemaps(self):
+ return [
+ "http://example.com/sitemap.xml", # already found below
+ "/sitemap-from-parser.xml", # only reported through the signal
+ ]
+
+ crawler.signals.send_catch_log(
+ signal=signals.robots_parsed,
+ robotparser=FakeRobotParser(),
+ request=Request("http://www.example.com/robots.txt"),
+ )
+
+ robots = b"Sitemap: http://example.com/sitemap.xml\n"
+ r = TextResponse(url="http://www.example.com/robots.txt", body=robots)
+ assert [req.url for req in spider._parse_sitemap(r)] == [
+ "http://example.com/sitemap.xml",
+ "http://www.example.com/sitemap-from-parser.xml",
+ ]
+
+ def test_robots_parsed_signal_not_connected_without_robotstxt_obey(self):
+ crawler = get_crawler(self.spider_class)
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+
+ crawler.signals.send_catch_log(
+ signal=signals.robots_parsed,
+ robotparser=None,
+ request=Request("http://www.example.com/robots.txt"),
+ )
+
+ assert spider._robots_sitemaps == {}
+
+ def test_robots_parsed_signal_schedules_sitemaps_for_skipped_netloc(self):
+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})
+ spider = self.spider_class.from_crawler(crawler, "example.com")
+ crawler.engine = mock.MagicMock()
+ spider._signal_only_netlocs.add("www.example.com")
+
+ class FakeRobotParser:
+ def sitemaps(self):
+ return ["/sitemap.xml"]
+
+ # request is whatever other request to that host triggered the fetch,
+ # not necessarily a request for robots.txt itself.
+ crawler.signals.send_catch_log(
+ signal=signals.robots_parsed,
+ robotparser=FakeRobotParser(),
+ request=Request("http://www.example.com/some/page"),
+ )
+
+ crawler.engine.crawl.assert_called_once()
+ scheduled = crawler.engine.crawl.call_args[0][0]
+ assert scheduled.url == "http://www.example.com/sitemap.xml"
+ assert scheduled.callback == spider._parse_sitemap # pylint: disable=comparison-with-callable
+ assert spider._signal_only_netlocs == set()
+ assert spider._robots_sitemaps == {}
+
def test_alternate_url_locs(self):
sitemap = b"""<?xml version="1.0" encoding="UTF-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"
@@ -356,6 +419,93 @@ Sitemap: /sitemap-relative-url.xml
items, _ = await crawl_items(_Spider, mockserver)
assert items == [{"url": mockserver.url("/text")}]
+ @coroutine_test
+ async def test_robots_txt_requested_once(self, mockserver: MockServer):
+ """robots.txt should be downloaded only once, even though both
+ SitemapSpider and RobotsTxtMiddleware need its content: the spider
+ requests it directly to look for sitemaps, while the middleware
+ downloads it to know what the spider is allowed to crawl.
+ """
+ robots_url = mockserver.url("/robots.txt")
+
+ class _Spider(self.spider_class): # type: ignore[name-defined,misc]
+ name = "robots_txt_requested_once"
+ custom_settings = {"ROBOTSTXT_OBEY": True}
+ sitemap_urls = [robots_url]
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.robots_txt_requests = []
+ crawler.signals.connect(
+ spider._track_downloader_hit,
+ signals.request_reached_downloader,
+ )
+ return spider
+
+ def _track_downloader_hit(self, request, spider):
+ if request.url == robots_url:
+ self.robots_txt_requests.append(request)
+
+ _, crawler = await crawl_items(_Spider, mockserver)
+ assert isinstance(crawler.spider, _Spider)
+ assert len(crawler.spider.robots_txt_requests) == 1
+
+ @coroutine_test
+ async def test_start_skips_robots_url_covered_by_another_sitemap_url(self):
+ crawler = get_crawler(self.spider_class, settings_dict={"ROBOTSTXT_OBEY": True})
+ spider = self.spider_class.from_crawler(
+ crawler,
+ "example.com",
+ sitemap_urls=[
+ "http://www.example.com/sitemap.xml",
+ "http://www.example.com/robots.txt",
+ "http://other.example.com/robots.txt",
+ ],
+ )
+
+ urls = [request.url async for request in spider.start()]
+
+ assert urls == [
+ "http://www.example.com/sitemap.xml",
+ "http://other.example.com/robots.txt",
+ ]
+ assert spider._signal_only_netlocs == {"www.example.com"}
+
+ @coroutine_test
+ async def test_robots_txt_requested_once_when_other_sitemap_url_goes_first(
+ self, mockserver: MockServer
+ ):
+ """When another sitemap_urls entry targets the same host, the spider
+ should not also request robots.txt itself: RobotsTxtMiddleware's own
+ robots.txt request, triggered by that other entry, covers it.
+ """
+ robots_url = mockserver.url("/robots.txt")
+ other_url = mockserver.url("/text")
+
+ class _Spider(self.spider_class): # type: ignore[name-defined,misc]
+ name = "robots_txt_requested_once_other_first"
+ custom_settings = {"ROBOTSTXT_OBEY": True}
+ sitemap_urls = [other_url, robots_url]
+
+ @classmethod
+ def from_crawler(cls, crawler, *args, **kwargs):
+ spider = super().from_crawler(crawler, *args, **kwargs)
+ spider.robots_txt_requests = []
+ crawler.signals.connect(
+ spider._track_downloader_hit,
+ signals.request_reached_downloader,
+ )
+ return spider
+
+ def _track_downloader_hit(self, request, spider):
+ if request.url == robots_url:
+ self.robots_txt_requests.append(request)
+
+ _, crawler = await crawl_items(_Spider, mockserver)
+ assert isinstance(crawler.spider, _Spider)
+ assert len(crawler.spider.robots_txt_requests) == 1
+
def test_parse_sitemap_empty_body(self, caplog: pytest.LogCaptureFixture) -> None:
r = XmlResponse(url="http://www.example.com/sitemap.xml", body=b"")
spider = self.spider_class("example.com")