Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support per-request download handlers (#7885)

Adriandcb849de83f3
diff --git a/docs/topics/download-handlers.rst b/docs/topics/download-handlers.rst
index b3ab386a7..19e5de2e3 100644
--- a/docs/topics/download-handlers.rst
+++ b/docs/topics/download-handlers.rst
@@ -43,6 +43,48 @@ for additional schemes and to replace or disable default ones:
:ref:`security-local-resources`, for the security implications of the
default ``http``, ``ftp``, ``file`` and ``data`` handlers.
+.. _download-handler-ids:
+
+Choosing a download handler per request
+---------------------------------------
+
+.. versionadded:: VERSION
+
+The :setting:`DOWNLOAD_HANDLERS_BY_NAME` setting registers handlers under a
+name instead of a URL scheme. Such handlers are only used by requests that ask
+for them through the :reqmeta:`download_handler` metadata key:
+
+.. code-block:: python
+
+ DOWNLOAD_HANDLERS_BY_NAME = {
+ "playwright": "scrapy_playwright.handler.ScrapyPlaywrightDownloadHandler",
+ }
+
+.. invisible-code-block: python
+
+ from scrapy import Request
+
+.. code-block:: python
+
+ Request("https://example.com", meta={"download_handler": "playwright"})
+
+:reqmeta:`download_handler` also accepts a URL scheme, e.g. ``"https"``, to use
+a handler from :setting:`DOWNLOAD_HANDLERS` on a request whose URL uses a
+different scheme.
+
+Requests keep their :reqmeta:`download_handler` across redirects and retries,
+and :ref:`robots.txt <topics-dlmw-robots>` requests inherit it from the request
+that triggers them. To fetch ``robots.txt`` differently, use a :ref:`downloader
+middleware <topics-downloader-middleware>` that acts on requests with the
+:reqmeta:`is_robotstxt_request` metadata key:
+
+.. code-block:: python
+
+ class DirectRobotsTxtMiddleware:
+ def process_request(self, request):
+ if request.meta.get("is_robotstxt_request"):
+ request.meta.pop("download_handler", None)
+
Replacing HTTP(S) download handlers
-----------------------------------
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index b11d2c76e..c606cc02b 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -889,6 +889,17 @@ This meta key is not supported by
:class:`~scrapy.core.downloader.handlers._aiohttp.AiohttpDownloadHandler`, but
the :setting:`DOWNLOAD_BIND_ADDRESS` setting is supported by them.
+.. reqmeta:: download_handler
+
+download_handler
+----------------
+
+.. versionadded:: VERSION
+
+ID of the :ref:`download handler <topics-download-handlers>` to use for this
+request, either a name from :setting:`DOWNLOAD_HANDLERS_BY_NAME` or a URL
+scheme from :setting:`DOWNLOAD_HANDLERS`. See :ref:`download-handler-ids`.
+
.. reqmeta:: download_timeout
download_timeout
@@ -963,6 +974,18 @@ http_user
Overrides :setting:`HTTPAUTH_USER` for this request.
+.. reqmeta:: is_robotstxt_request
+
+is_robotstxt_request
+--------------------
+
+.. versionadded:: VERSION
+
+``True`` on the ``robots.txt`` requests that
+:class:`~scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware` sends, so
+that :ref:`downloader middlewares <topics-downloader-middleware>` can tell them
+apart from other requests.
+
.. reqmeta:: max_retry_times
max_retry_times
diff --git a/docs/topics/settings.rst b/docs/topics/settings.rst
index c396fb1c1..afa587fb3 100644
--- a/docs/topics/settings.rst
+++ b/docs/topics/settings.rst
@@ -1145,6 +1145,19 @@ handler (without replacement), place this in your ``settings.py``:
:ref:`security-local-resources`
+.. setting:: DOWNLOAD_HANDLERS_BY_NAME
+
+DOWNLOAD_HANDLERS_BY_NAME
+-------------------------
+
+.. versionadded:: VERSION
+
+Default: ``{}``
+
+A dict mapping names to :ref:`download handlers <topics-download-handlers>`
+that requests can ask for by name. See :ref:`download-handler-ids`.
+
+
.. reqmeta:: download_slot
.. setting:: DOWNLOAD_SLOTS
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.py
index 57a8e962b..68db7ed49 100644
--- a/scrapy/core/downloader/handlers/__init__.py
+++ b/scrapy/core/downloader/handlers/__init__.py
@@ -61,44 +61,59 @@ class DownloadHandlerProtocol(Protocol):
class DownloadHandlers:
def __init__(self, crawler: Crawler):
self._crawler: Crawler = crawler
- # stores acceptable schemes on instancing
- self._schemes: dict[str, str | Callable[..., Any]] = {}
- # stores instanced handlers for schemes
+ # stores class paths by handler ID
+ self._paths: dict[str, str | Callable[..., Any]] = {}
+ # handler IDs that are also acceptable URL schemes
+ self._schemes: set[str] = set()
+ # stores instanced handlers by handler ID
self._handlers: dict[str, DownloadHandlerProtocol] = {}
# remembers failed handlers
self._notconfigured: dict[str, str] = {}
# remembers handlers with Deferred-based download_request()
self._old_style_handlers: set[str] = set()
- handlers: dict[str, str | Callable[..., Any]] = without_none_values(
+ scheme_handlers: dict[str, str | Callable[..., Any]] = without_none_values(
cast(
"dict[str, str | Callable[..., Any]]",
crawler.settings.getwithbase("DOWNLOAD_HANDLERS"),
)
)
- for scheme, clspath in handlers.items():
- self._schemes[scheme] = clspath
- self._load_handler(scheme, skip_lazy=True)
+ named_handlers: dict[str, str | Callable[..., Any]] = without_none_values(
+ cast(
+ "dict[str, str | Callable[..., Any]]",
+ crawler.settings.getdict("DOWNLOAD_HANDLERS_BY_NAME"),
+ )
+ )
+ if clashes := scheme_handlers.keys() & named_handlers.keys():
+ raise ValueError(
+ f"The following download handler IDs are defined both in "
+ f"DOWNLOAD_HANDLERS and in DOWNLOAD_HANDLERS_BY_NAME: "
+ f"{', '.join(sorted(clashes))}."
+ )
+ self._schemes.update(scheme_handlers)
+ for handler_id, clspath in (scheme_handlers | named_handlers).items():
+ self._paths[handler_id] = clspath
+ self._load_handler(handler_id, skip_lazy=True)
crawler.signals.connect(self._close, signals.engine_stopped)
- def _get_handler(self, scheme: str) -> DownloadHandlerProtocol | None:
- """Lazy-load the downloadhandler for a scheme
- only on the first request for that scheme.
+ def _get_handler(self, handler_id: str) -> DownloadHandlerProtocol | None:
+ """Lazy-load the download handler with the given ID only on its first
+ request.
"""
- if scheme in self._handlers:
- return self._handlers[scheme]
- if scheme in self._notconfigured:
+ if handler_id in self._handlers:
+ return self._handlers[handler_id]
+ if handler_id in self._notconfigured:
return None
- if scheme not in self._schemes:
- self._notconfigured[scheme] = "no handler available for that scheme"
+ if handler_id not in self._paths:
+ self._notconfigured[handler_id] = "no handler with that ID"
return None
- return self._load_handler(scheme)
+ return self._load_handler(handler_id)
def _load_handler(
- self, scheme: str, skip_lazy: bool = False
+ self, handler_id: str, skip_lazy: bool = False
) -> DownloadHandlerProtocol | None:
- path = self._schemes[scheme]
+ path = self._paths[handler_id]
try:
dhcls: type[DownloadHandlerProtocol] = load_object(path)
if skip_lazy:
@@ -117,17 +132,17 @@ class DownloadHandlers:
self._crawler,
)
except NotConfigured as ex:
- self._notconfigured[scheme] = str(ex)
+ self._notconfigured[handler_id] = str(ex)
return None
except Exception as ex:
logger.exception(
- 'Loading "%(clspath)s" for scheme "%(scheme)s"',
- {"clspath": path, "scheme": scheme},
+ 'Loading "%(clspath)s" for handler ID "%(handler_id)s"',
+ {"clspath": path, "handler_id": handler_id},
extra={"crawler": self._crawler},
)
- self._notconfigured[scheme] = str(ex)
+ self._notconfigured[handler_id] = str(ex)
return None
- self._handlers[scheme] = dh
+ self._handlers[handler_id] = dh
if not inspect.iscoroutinefunction(dh.download_request): # pragma: no cover
warnings.warn(
f"{global_object_name(dh.download_request)} is not a coroutine function."
@@ -136,7 +151,7 @@ class DownloadHandlers:
category=ScrapyDeprecationWarning,
stacklevel=1,
)
- self._old_style_handlers.add(scheme)
+ self._old_style_handlers.add(handler_id)
return dh
def download_request(
@@ -150,14 +165,33 @@ class DownloadHandlers:
return deferred_from_coro(self.download_request_async(request))
async def download_request_async(self, request: Request) -> Response:
- scheme = urlparse_cached(request).scheme
- handler = self._get_handler(scheme)
- if not handler:
- raise NotSupported(
- f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
- )
+ handler_id = request.meta.get("download_handler")
+ if handler_id is None:
+ scheme = urlparse_cached(request).scheme
+ # Handler IDs from DOWNLOAD_HANDLERS_BY_NAME are only reachable
+ # through the download_handler request metadata key, so that
+ # registering one does not make a new URL scheme downloadable.
+ if scheme in self._schemes:
+ handler = self._get_handler(scheme)
+ else:
+ handler = None
+ self._notconfigured.setdefault(
+ scheme, "no handler available for that scheme"
+ )
+ if not handler:
+ raise NotSupported(
+ f"Unsupported URL scheme '{scheme}': {self._notconfigured[scheme]}"
+ )
+ handler_id = scheme
+ else:
+ handler = self._get_handler(handler_id)
+ if not handler:
+ raise NotSupported(
+ f"Unusable download handler {handler_id!r}: "
+ f"{self._notconfigured[handler_id]}"
+ )
assert self._crawler.spider
- if scheme in self._old_style_handlers: # pragma: no cover
+ if handler_id in self._old_style_handlers: # pragma: no cover
return await maybe_deferred_to_future(
cast(
"Deferred[Response]",
diff --git a/scrapy/downloadermiddlewares/robotstxt.py b/scrapy/downloadermiddlewares/robotstxt.py
index 38659024e..210d52586 100644
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@ -7,7 +7,7 @@ enable this middleware and enable the ROBOTSTXT_OBEY setting.
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING, Self
+from typing import TYPE_CHECKING, Any, Self
from twisted.internet.defer import Deferred
@@ -87,10 +87,16 @@ class RobotsTxtMiddleware:
if netloc not in self._parsers:
self._parsers[netloc] = Deferred()
robotsurl = f"{url.scheme}://{url.netloc}/robots.txt"
+ meta: dict[str, Any] = {
+ "dont_obey_robotstxt": True,
+ "is_robotstxt_request": True,
+ }
+ if "download_handler" in request.meta:
+ meta["download_handler"] = request.meta["download_handler"]
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
- meta={"dont_obey_robotstxt": True},
+ meta=meta,
callback=NO_CALLBACK,
)
try:
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index c215e23ce..62a32acf4 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -73,6 +73,7 @@ __all__ = [
"DOWNLOAD_FAIL_ON_DATALOSS",
"DOWNLOAD_HANDLERS",
"DOWNLOAD_HANDLERS_BASE",
+ "DOWNLOAD_HANDLERS_BY_NAME",
"DOWNLOAD_MAXSIZE",
"DOWNLOAD_SLOTS",
"DOWNLOAD_TIMEOUT",
@@ -310,6 +311,7 @@ DOWNLOAD_HANDLERS_BASE = {
"s3": "scrapy.core.downloader.handlers.s3.S3DownloadHandler",
"ftp": "scrapy.core.downloader.handlers.ftp.FTPDownloadHandler",
}
+DOWNLOAD_HANDLERS_BY_NAME: dict[str, str] = {}
DOWNLOAD_MAXSIZE = 1024 * 1024 * 1024 # 1024m
DOWNLOAD_WARNSIZE = 32 * 1024 * 1024 # 32m
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 6a75f8193..248b4cd28 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -29,7 +29,7 @@ logger = logging.getLogger(__name__)
_fingerprint_cache: WeakKeyDictionary[
- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes]
] = WeakKeyDictionary()
@@ -78,8 +78,18 @@ def fingerprint(
)
verbatim_url = bool(request.meta.get("verbatim_url"))
effective_keep_fragments = keep_fragments and not verbatim_url
+ # A handler ID matching the URL scheme is the one that would be used
+ # anyway, so it is left out to keep such fingerprints unchanged.
+ handler_id: str | None = request.meta.get("download_handler")
+ if handler_id is not None and handler_id == urlparse_cached(request).scheme:
+ handler_id = None
cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)
+ cache_key = (
+ processed_include_headers,
+ effective_keep_fragments,
+ verbatim_url,
+ handler_id,
+ )
if cache_key not in cache:
# To decode bytes reliably (JSON does not support bytes), regardless of
# character encoding, we use bytes.hex()
@@ -101,6 +111,8 @@ def fingerprint(
"body": (request.body or b"").hex(),
"headers": headers,
}
+ if handler_id is not None:
+ fingerprint_data["download_handler"] = handler_id
fingerprint_json = json.dumps(fingerprint_data, sort_keys=True)
cache[cache_key] = hashlib.sha1( # noqa: S324
fingerprint_json.encode()
@@ -115,13 +127,17 @@ class RequestFingerprinterProtocol(Protocol):
class RequestFingerprinter:
"""Default fingerprinter.
+ .. versionchanged:: VERSION
+ :reqmeta:`download_handler` is taken into account.
+
It takes into account a canonical version
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
<scrapy.Request.url>` and the values of :attr:`request.method
<scrapy.Request.method>` and :attr:`request.body
<scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that
- request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_
- hash.
+ request. It also takes into account :reqmeta:`download_handler` when it
+ does not match the URL scheme. It then generates an `SHA1
+ <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
"""
@classmethod
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index ad77fd7dd..ca9860271 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -28,10 +28,11 @@ from scrapy.exceptions import (
DownloadFailedError,
DownloadTimeoutError,
NotConfigured,
+ NotSupported,
ScrapyDeprecationWarning,
UnsupportedURLSchemeError,
)
-from scrapy.http import Request, TextResponse
+from scrapy.http import Request, Response, TextResponse
from scrapy.responsetypes import responsetypes
from scrapy.utils._download_handlers import wrap_twisted_exceptions
from scrapy.utils.boto import is_botocore_available
@@ -47,6 +48,20 @@ class DummyDH:
pass
+class NamedDH:
+ lazy = False
+
+ async def download_request(self, request):
+ return Response(request.url, body=b"named")
+
+
+class SchemeDH:
+ lazy = False
+
+ async def download_request(self, request):
+ return Response(request.url, body=b"scheme")
+
+
class DummyLazyDH:
# Default (but deprecated) is lazy for backward compatibility
async def download_request(self, request):
@@ -100,7 +115,7 @@ class TestLoad:
assert "scheme" not in dh._handlers
assert "scheme" in dh._notconfigured
assert (
- 'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for scheme "scheme"'
+ 'Loading "<class \'tests.test_downloader_handlers.BuggyDH\'>" for handler ID "scheme"'
in caplog.text
)
@@ -131,6 +146,58 @@ class TestLoad:
assert "scheme" not in dh._notconfigured
+class TestHandlerID:
+ @staticmethod
+ def _get_dh() -> DownloadHandlers:
+ crawler = get_crawler(
+ settings_dict={
+ "DOWNLOAD_HANDLERS": {"https": SchemeDH},
+ "DOWNLOAD_HANDLERS_BY_NAME": {"named": NamedDH},
+ }
+ )
+ crawler.spider = crawler._create_spider()
+ return DownloadHandlers(crawler)
+
+ def test_clashing_ids(self) -> None:
+ crawler = get_crawler(
+ settings_dict={
+ "DOWNLOAD_HANDLERS": {"https": SchemeDH},
+ "DOWNLOAD_HANDLERS_BY_NAME": {"https": NamedDH},
+ }
+ )
+ with pytest.raises(ValueError, match="defined both in DOWNLOAD_HANDLERS"):
+ DownloadHandlers(crawler)
+
+ @coroutine_test
+ async def test_name_not_a_scheme(self) -> None:
+ dh = self._get_dh()
+ with pytest.raises(NotSupported, match="Unsupported URL scheme 'named'"):
+ await dh.download_request_async(Request("named://example.com"))
+
+ @coroutine_test
+ async def test_name_in_meta(self) -> None:
+ dh = self._get_dh()
+ request = Request("https://example.com", meta={"download_handler": "named"})
+ response = await dh.download_request_async(request)
+ assert response.body == b"named"
+
+ @coroutine_test
+ async def test_scheme_in_meta(self) -> None:
+ dh = self._get_dh()
+ request = Request("ftp://example.com", meta={"download_handler": "https"})
+ response = await dh.download_request_async(request)
+ assert response.body == b"scheme"
+
+ @coroutine_test
+ async def test_unknown_id_in_meta(self) -> None:
+ dh = self._get_dh()
+ request = Request("https://example.com", meta={"download_handler": "nmaed"})
+ with pytest.raises(
+ NotSupported, match="Unusable download handler 'nmaed': no handler with"
+ ):
+ await dh.download_request_async(request)
+
+
class TestFile:
def setup_method(self):
# add a special char to check that they are handled correctly
diff --git a/tests/test_downloadermiddleware_robotstxt.py b/tests/test_downloadermiddleware_robotstxt.py
index f55e57b4f..62b91e9f4 100644
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@ -275,6 +275,17 @@ Disallow: /some/randome/page.html
await middleware.process_request(Request("http://site.local/allowed"))
assert middleware.process_request_2.called
+ @coroutine_test
+ async def test_robotstxt_download_handler(self) -> None:
+ middleware = RobotsTxtMiddleware(self._get_successful_crawler())
+ await self.assertNotIgnored(
+ Request("http://site.local/allowed", meta={"download_handler": "named"}),
+ middleware,
+ )
+ robotsreq = self.crawler.engine.download_async.call_args_list[0][0][0]
+ assert robotsreq.meta["download_handler"] == "named"
+ assert robotsreq.meta["is_robotstxt_request"] is True
+
async def assertNotIgnored(
self, request: Request, middleware: RobotsTxtMiddleware
) -> None:
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 32130ded8..ce0ac8ff5 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -77,13 +77,15 @@ class TestFingerprint:
function: _FingerprintFunction = staticmethod(fingerprint)
cache: (
WeakKeyDictionary[
- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
+ Request,
+ dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], bytes],
]
| WeakKeyDictionary[
- Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]
+ Request,
+ dict[tuple[tuple[bytes, ...] | None, bool, bool, str | None], str],
]
) = _fingerprint_cache
- default_cache_key = (None, False, False)
+ default_cache_key = (None, False, False, None)
known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
(
Request("http://example.org"),
@@ -239,6 +241,39 @@ class TestFingerprint:
assert self.function(r5) == self.function(r5, keep_fragments=True)
assert self.function(r5) == self.function(r5, keep_fragments=False)
+ def test_download_handler(self):
+ r1 = Request("https://example.com")
+ r2 = Request("https://example.com", meta={"download_handler": "playwright"})
+ r3 = Request("https://example.com", meta={"download_handler": "http"})
+ assert self.function(r1) != self.function(r2)
+ assert self.function(r2) != self.function(r3)
+
+ # An ID matching the URL scheme is the handler that would be used
+ # anyway, so it does not change the fingerprint.
+ r4 = Request("https://example.com", meta={"download_handler": "https"})
+ assert self.function(r1) == self.function(r4)
+
+ # IDs are case-sensitive, and URL schemes are lowercased.
+ r5 = Request("HTTPS://example.com", meta={"download_handler": "HTTPS"})
+ assert self.function(r1) != self.function(r5)
+
+ def test_download_handler_caching(self):
+ # The cached fingerprint must account for the handler ID, which can
+ # change after the first call.
+ r1 = Request("https://example.com")
+ fp1 = self.function(r1)
+ r1.meta["download_handler"] = "playwright"
+ assert self.function(r1) != fp1
+
+ def test_download_handler_and_verbatim_url(self):
+ r1 = Request(
+ "https://example.com/a b",
+ meta={"verbatim_url": True, "download_handler": "playwright"},
+ )
+ r2 = Request("https://example.com/a b", meta={"verbatim_url": True})
+ r3 = Request("https://example.com/a b", meta={"download_handler": "playwright"})
+ assert len({self.function(r1), self.function(r2), self.function(r3)}) == 3
+
def test_method_and_body(self):
r1 = Request("http://www.example.com")
r2 = Request("http://www.example.com", method="POST")