Scrapy, a fast high-level web crawling & scraping framework for Python.
Support per-request download handlers (#7885)
Adriandcb849de83f3
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rstindex b3ab386a7..19e5de2e3 100644--- a/docs/topics/download-handlers.rst+++ b/docs/topics/download-handlers.rst@@ -43,6 +43,48 @@ for additional schemes and to replace or disable default ones::ref:`security-local-resources`, for the security implications of thedefault ``http``, ``ftp``, ``file`` and ``data`` handlers.+.. _download-handler-ids:++Choosing a download handler per request+---------------------------------------++.. versionadded:: VERSION++The :setting:`DOWNLOAD_HANDLERS_BY_NAME` setting registers handlers under a+name instead of a URL scheme. Such handlers are only used by requests that ask+for them through the :reqmeta:`download_handler` metadata key:++.. code-block:: python++ DOWNLOAD_HANDLERS_BY_NAME = {+ "playwright": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",+ }++.. invisible-code-block: python++ from scrapy import Request++.. code-block:: python++ Request("https://example.com", meta={"download_handler": "playwright"})++:reqmeta:`download_handler` also accepts a URL scheme, e.g. ``"https"``, to use+a handler from :setting:`DOWNLOAD_HANDLERS` on a request whose URL uses a+different scheme.++Requests keep their :reqmeta:`download_handler` across redirects and retries,+and :ref:`robots.txt <topics-dlmw-robots>` requests inherit it from the request+that triggers them. To fetch ``robots.txt`` differently, use a :ref:`downloader+middleware <topics-downloader-middleware>` that acts on requests with the+:reqmeta:`is_robotstxt_request` metadata key:++.. code-block:: python++ class DirectRobotsTxtMiddleware:+ def process_request(self, request):+ if request.meta.get("is_robotstxt_request"):+ request.meta.pop("download_handler", None)+Replacing HTTP(S) download handlers-----------------------------------diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex b11d2c76e..c606cc02b 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -889,6 +889,17 @@ This meta key is not supported by:class:`~scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler`, butthe :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported by them.+.. reqmeta:: download_handler++download_handler+----------------++.. versionadded:: VERSION++ID of the :ref:`download handler <topics-download-handlers>` to use for this+request, either a name from :setting:`DOWNLOAD_HANDLERS_BY_NAME` or a URL+scheme from :setting:`DOWNLOAD_HANDLERS`. See :ref:`download-handler-ids`.+.. reqmeta:: download_timeoutdownload_timeout@@ -963,6 +974,18 @@ http_userOverrides :setting:`HTTPAUTH_USER` for this request.+.. reqmeta:: is_robotstxt_request++is_robotstxt_request+--------------------++.. versionadded:: VERSION++``True`` on the ``robots.txt`` requests that+:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` sends, so+that :ref:`downloader middlewares <topics-downloader-middleware>` can tell them+apart from other requests.+.. reqmeta:: max_retry_timesmax_retry_timesdiff --git a/docs/topics/settings.rst b/docs/topics/settings.rstindex c396fb1c1..afa587fb3 100644--- a/docs/topics/settings.rst+++ b/docs/topics/settings.rst@@ -1145,6 +1145,19 @@ handler (without replacement), place this in your ``settings.py``::ref:`security-local-resources`+.. setting:: DOWNLOAD_HANDLERS_BY_NAME++DOWNLOAD_HANDLERS_BY_NAME+-------------------------++.. versionadded:: VERSION++Default: ``{}``++A dict mapping names to :ref:`download handlers <topics-download-handlers>`+that requests can ask for by name. See :ref:`download-handler-ids`.++.. reqmeta:: download_slot.. setting:: DOWNLOAD_SLOTSdiff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 57a8e962b..68db7ed49 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -61,44 +61,59 @@ class DownloadHandlerProtocol(Protocol):class DownloadHandlers:def __init__(self, crawler: Crawler):self._crawler: Crawler = crawler- # stores acceptable schemes on instancing- self._schemes: dict[str, str | Callable[..., Any]] = {}- # stores instanced handlers for schemes+ # stores class paths by handler ID+ self._paths: dict[str, str | Callable[..., Any]] = {}+ # handler IDs that are also acceptable URL schemes+ self._schemes: set[str] = set()+ # stores instanced handlers by handler IDself._handlers: dict[str, DownloadHandlerProtocol] = {}# remembers failed handlersself._notconfigured: dict[str, str] = {}# remembers handlers with Deferred-based download_request()self._old_style_handlers: set[str] = set()- handlers: dict[str, str | Callable[..., Any]] = without_none_values(+ scheme_handlers: dict[str, str | Callable[..., Any]] = without_none_values(cast("dict[str, str | Callable[..., Any]]",crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),))- for scheme, clspath in handlers.items():- self._schemes[scheme] = clspath- self._load_handler(scheme, skip_lazy=True)+ named_handlers: dict[str, str | Callable[..., Any]] = without_none_values(+ cast(+ "dict[str, str | Callable[..., Any]]",+ crawler.settings.getdict("DOWNLOAD_HANDLERS_BY_NAME"),+ )+ )+ if clashes := scheme_handlers.keys() & named_handlers.keys():+ raise ValueError(+ f"The following download handler IDs are defined both in "+ f"DOWNLOAD_HANDLERS and in DOWNLOAD_HANDLERS_BY_NAME: "+ f"{', '.join(sorted(clashes))}."+ )+ self._schemes.update(scheme_handlers)+ for handler_id, clspath in (scheme_handlers | named_handlers).items():+ self._paths[handler_id] = clspath+ self._load_handler(handler_id, skip_lazy=True)crawler.signals.connect(self._close, signals.engine_stopped)- def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None:- """Lazy-load the downloadhandler for a scheme- only on the first request for that scheme.+ def _get_handler(self, handler_id: str) -> DownloadHandlerProtocol | None:+ """Lazy-load the download handler with the given ID only on its first+ request."""- if scheme in self._handlers:- return self._handlers[scheme]- if scheme in self._notconfigured:+ if handler_id in self._handlers:+ return self._handlers[handler_id]+ if handler_id in self._notconfigured:return None- if scheme not in self._schemes:- self._notconfigured[scheme] = "no handler available for that scheme"+ if handler_id not in self._paths:+ self._notconfigured[handler_id] = "no handler with that ID"return None- return self._load_handler(scheme)+ return self._load_handler(handler_id)def _load_handler(- self, scheme: str, skip_lazy: bool = False+ self, handler_id: str, skip_lazy: bool = False) -> DownloadHandlerProtocol | None:- path = self._schemes[scheme]+ path = self._paths[handler_id]try:dhcls: type[DownloadHandlerProtocol] = load_object(path)if skip_lazy:@@ -117,17 +132,17 @@ class DownloadHandlers:self._crawler,)except NotConfigured as ex:- self._notconfigured[scheme] = str(ex)+ self._notconfigured[handler_id] = str(ex)return Noneexcept Exception as ex:logger.exception(- 'Loading "%(clspath)s" for scheme "%(scheme)s"',- {"clspath": path, "scheme": scheme},+ 'Loading "%(clspath)s" for handler ID "%(handler_id)s"',+ {"clspath": path, "handler_id": handler_id},extra={"crawler": self._crawler},)- self._notconfigured[scheme] = str(ex)+ self._notconfigured[handler_id] = str(ex)return None- self._handlers[scheme] = dh+ self._handlers[handler_id] = dhif not inspect.iscoroutinefunction(dh.download_request): # pragma: no coverwarnings.warn(f"{global_object_name(dh.download_request)} is not a coroutine function."@@ -136,7 +151,7 @@ class DownloadHandlers:category=ScrapyDeprecationWarning,stacklevel=1,)- self._old_style_handlers.add(scheme)+ self._old_style_handlers.add(handler_id)return dhdef download_request(@@ -150,14 +165,33 @@ class DownloadHandlers:return deferred_from_coro(self.download_request_async(request))async def download_request_async(self, request: Request) -> Response:- scheme = urlparse_cached(request).scheme- handler = self._get_handler(scheme)- if not handler:- raise NotSupported(- f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"- )+ handler_id = request.meta.get("download_handler")+ if handler_id is None:+ scheme = urlparse_cached(request).scheme+ # Handler IDs from DOWNLOAD_HANDLERS_BY_NAME are only reachable+ # through the download_handler request metadata key, so that+ # registering one does not make a new URL scheme downloadable.+ if scheme in self._schemes:+ handler = self._get_handler(scheme)+ else:+ handler = None+ self._notconfigured.setdefault(+ scheme, "no handler available for that scheme"+ )+ if not handler:+ raise NotSupported(+ f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"+ )+ handler_id = scheme+ else:+ handler = self._get_handler(handler_id)+ if not handler:+ raise NotSupported(+ f"Unusable download handler {handler_id!r}: "+ f"{self._notconfigured[handler_id]}"+ )assert self._crawler.spider- if scheme in self._old_style_handlers: # pragma: no cover+ if handler_id in self._old_style_handlers: # pragma: no coverreturn await maybe_deferred_to_future(cast("Deferred[Response]",diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.pyindex 38659024e..210d52586 100644--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.from __future__ import annotationsimport logging-from typing import TYPE_CHECKING, Self+from typing import TYPE_CHECKING, Any, Selffrom twisted.internet.defer import Deferred@@ -87,10 +87,16 @@ class RobotsTxtMiddleware:if netloc not in self._parsers:self._parsers[netloc] = Deferred()robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"+ meta: dict[str, Any] = {+ "dont_obey_robotstxt": True,+ "is_robotstxt_request": True,+ }+ if "download_handler" in request.meta:+ meta["download_handler"] = request.meta["download_handler"]robotsreq = Request(robotsurl,priority=self.DOWNLOAD_PRIORITY,- meta={"dont_obey_robotstxt": True},+ meta=meta,callback=NO_CALLBACK,)try:diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex c215e23ce..62a32acf4 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -73,6 +73,7 @@ __all__ = ["DOWNLOAD_FAIL_ON_DATALOSS","DOWNLOAD_HANDLERS","DOWNLOAD_HANDLERS_BASE",+ "DOWNLOAD_HANDLERS_BY_NAME","DOWNLOAD_MAXSIZE","DOWNLOAD_SLOTS","DOWNLOAD_TIMEOUT",@@ -310,6 +311,7 @@ DOWNLOAD_HANDLERS_BASE = {"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler","ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",}+DOWNLOAD_HANDLERS_BY_NAME: dict[str, str] = {}DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024mDOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32mdiff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 6a75f8193..248b4cd28 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)_fingerprint_cache: WeakKeyDictionary[- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes]] = WeakKeyDictionary()@@ -78,8 +78,18 @@ def fingerprint()verbatim_url = bool(request.meta.get("verbatim_url"))effective_keep_fragments = keep_fragments and not verbatim_url+ # A handler ID matching the URL scheme is the one that would be used+ # anyway, so it is left out to keep such fingerprints unchanged.+ handler_id: str | None = request.meta.get("download_handler")+ if handler_id is not None and handler_id == urlparse_cached(request).scheme:+ handler_id = Nonecache = _fingerprint_cache.setdefault(request, {})- cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)+ cache_key = (+ processed_include_headers,+ effective_keep_fragments,+ verbatim_url,+ handler_id,+ )if cache_key not in cache:# To decode bytes reliably (JSON does not support bytes), regardless of# character encoding, we use bytes.hex()@@ -101,6 +111,8 @@ def fingerprint("body": (request.body or b"").hex(),"headers": headers,}+ if handler_id is not None:+ fingerprint_data["download_handler"] = handler_idfingerprint_json = json.dumps(fingerprint_data, sort_keys=True)cache[cache_key] = hashlib.sha1( # noqa: S324fingerprint_json.encode()@@ -115,13 +127,17 @@ class RequestFingerprinterProtocol(Protocol):class RequestFingerprinter:"""Default fingerprinter.+ .. versionchanged:: VERSION+ :reqmeta:`download_handler` is taken into account.+It takes into account a canonical version(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url<scrapy.Request.url>` and the values of :attr:`request.method<scrapy.Request.method>` and :attr:`request.body<scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that- request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_- hash.+ request. It also takes into account :reqmeta:`download_handler` when it+ does not match the URL scheme. It then generates an `SHA1+ <https://en.wikipedia.org/wiki/SHA-1>`_ hash."""@classmethoddiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex ad77fd7dd..ca9860271 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -28,10 +28,11 @@ from scrapy.exceptions import (DownloadFailedError,DownloadTimeoutError,NotConfigured,+ NotSupported,ScrapyDeprecationWarning,UnsupportedURLSchemeError,)-from scrapy.http import Request, TextResponse+from scrapy.http import Request, Response, TextResponsefrom scrapy.responsetypes import responsetypesfrom scrapy.utils._download_handlers import wrap_twisted_exceptionsfrom scrapy.utils.boto import is_botocore_available@@ -47,6 +48,20 @@ class DummyDH:pass+class NamedDH:+ lazy = False++ async def download_request(self, request):+ return Response(request.url, body=b"named")+++class SchemeDH:+ lazy = False++ async def download_request(self, request):+ return Response(request.url, body=b"scheme")++class DummyLazyDH:# Default (but deprecated) is lazy for backward compatibilityasync def download_request(self, request):@@ -100,7 +115,7 @@ class TestLoad:assert "scheme" not in dh._handlersassert "scheme" in dh._notconfiguredassert (- 'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for scheme "scheme"'+ 'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for handler ID "scheme"'in caplog.text)@@ -131,6 +146,58 @@ class TestLoad:assert "scheme" not in dh._notconfigured+class TestHandlerID:+ @staticmethod+ def _get_dh() -> DownloadHandlers:+ crawler = get_crawler(+ settings_dict={+ "DOWNLOAD_HANDLERS": {"https": SchemeDH},+ "DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH},+ }+ )+ crawler.spider = crawler._create_spider()+ return DownloadHandlers(crawler)++ def test_clashing_ids(self) -> None:+ crawler = get_crawler(+ settings_dict={+ "DOWNLOAD_HANDLERS": {"https": SchemeDH},+ "DOWNLOAD_HANDLERS_BY_NAME": {"https": NamedDH},+ }+ )+ with pytest.raises(ValueError, match="defined both in DOWNLOAD_HANDLERS"):+ DownloadHandlers(crawler)++ @coroutine_test+ async def test_name_not_a_scheme(self) -> None:+ dh = self._get_dh()+ with pytest.raises(NotSupported, match="Unsupported URL scheme 'named'"):+ await dh.download_request_async(Request("named://example.com"))++ @coroutine_test+ async def test_name_in_meta(self) -> None:+ dh = self._get_dh()+ request = Request("https://example.com", meta={"download_handler": "named"})+ response = await dh.download_request_async(request)+ assert response.body == b"named"++ @coroutine_test+ async def test_scheme_in_meta(self) -> None:+ dh = self._get_dh()+ request = Request("ftp://example.com", meta={"download_handler": "https"})+ response = await dh.download_request_async(request)+ assert response.body == b"scheme"++ @coroutine_test+ async def test_unknown_id_in_meta(self) -> None:+ dh = self._get_dh()+ request = Request("https://example.com", meta={"download_handler": "nmaed"})+ with pytest.raises(+ NotSupported, match="Unusable download handler 'nmaed': no handler with"+ ):+ await dh.download_request_async(request)++class TestFile:def setup_method(self):# add a special char to check that they are handled correctlydiff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.pyindex f55e57b4f..62b91e9f4 100644--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@ -275,6 +275,17 @@ Disallow: /some/randome/page.htmlawait middleware.process_request(Request("http://site.local/allowed"))assert middleware.process_request_2.called+ @coroutine_test+ async def test_robotstxt_download_handler(self) -> None:+ middleware = RobotsTxtMiddleware(self._get_successful_crawler())+ await self.assertNotIgnored(+ Request("http://site.local/allowed", meta={"download_handler": "named"}),+ middleware,+ )+ robotsreq = self.crawler.engine.download_async.call_args_list[0][0][0]+ assert robotsreq.meta["download_handler"] == "named"+ assert robotsreq.meta["is_robotstxt_request"] is True+async def assertNotIgnored(self, request: Request, middleware: RobotsTxtMiddleware) -> None:diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 32130ded8..ce0ac8ff5 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -77,13 +77,15 @@ class TestFingerprint:function: _FingerprintFunction = staticmethod(fingerprint)cache: (WeakKeyDictionary[- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]+ Request,+ dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes],]| WeakKeyDictionary[- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]+ Request,+ dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], str],]) = _fingerprint_cache- default_cache_key = (None, False, False)+ default_cache_key = (None, False, False, None)known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ((Request("http://example.org"),@@ -239,6 +241,39 @@ class TestFingerprint:assert self.function(r5) == self.function(r5, keep_fragments=True)assert self.function(r5) == self.function(r5, keep_fragments=False)+ def test_download_handler(self):+ r1 = Request("https://example.com")+ r2 = Request("https://example.com", meta={"download_handler": "playwright"})+ r3 = Request("https://example.com", meta={"download_handler": "http"})+ assert self.function(r1) != self.function(r2)+ assert self.function(r2) != self.function(r3)++ # An ID matching the URL scheme is the handler that would be used+ # anyway, so it does not change the fingerprint.+ r4 = Request("https://example.com", meta={"download_handler": "https"})+ assert self.function(r1) == self.function(r4)++ # IDs are case-sensitive, and URL schemes are lowercased.+ r5 = Request("HTTPS://example.com", meta={"download_handler": "HTTPS"})+ assert self.function(r1) != self.function(r5)++ def test_download_handler_caching(self):+ # The cached fingerprint must account for the handler ID, which can+ # change after the first call.+ r1 = Request("https://example.com")+ fp1 = self.function(r1)+ r1.meta["download_handler"] = "playwright"+ assert self.function(r1) != fp1++ def test_download_handler_and_verbatim_url(self):+ r1 = Request(+ "https://example.com/a b",+ meta={"verbatim_url": True, "download_handler": "playwright"},+ )+ r2 = Request("https://example.com/a b", meta={"verbatim_url": True})+ r3 = Request("https://example.com/a b", meta={"download_handler": "playwright"})+ assert len({self.function(r1), self.function(r2), self.function(r3)}) == 3+def test_method_and_body(self):r1 = Request("http://www.example.com")r2 = Request("http://www.example.com", method="POST")