Scrapy, a fast high-level web crawling & scraping framework for Python.
Support sending requests with "unsafe" URLs (#7473)
Adrianb2d8b06be61d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex d1e4851d9..a4f031803 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -117,6 +117,9 @@ Request objects:param encoding: the encoding of this request (defaults to ``'utf-8'``).This encoding will be used to percent-encode the URL and to convert thebody to bytes (if given as a string).++ To disable URL percent-encoding for a request, use the+ :reqmeta:`verbatim_url` request meta key.:type encoding: str:param priority: sets :attr:`priority`, defaults to ``0``.@@ -136,9 +139,13 @@ Request objects.. attribute:: Request.url- A string containing the URL of this request. Keep in mind that this- attribute contains the escaped URL, so it can differ from the URL passed in- the ``__init__()`` method.+ A string containing the URL of this request.++ Keep in mind that this attribute contains the escaped URL, so it can+ differ from the URL passed in the ``__init__()`` method.++ If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as+ passed to ``__init__()``.This attribute is read-only. To change the URL of a Request use:meth:`replace`.@@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation:.. autofunction:: scrapy.utils.request.fingerprint+By default, request fingerprinting canonicalizes the request URL. If+:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not+canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is+effectively true).+For example, to take the value of a request header named ``X-ID`` intoaccount:@@ -710,6 +722,7 @@ Those are:* :reqmeta:`redirect_reasons`* :reqmeta:`redirect_urls`* :reqmeta:`referrer_policy`+* :reqmeta:`verbatim_url`.. reqmeta:: bindaddress@@ -786,6 +799,21 @@ The meta key is used set retry times per request. When initialized, the:reqmeta:`max_retry_times` meta key takes higher precedence over the:setting:`RETRY_TIMES` setting.+.. reqmeta:: verbatim_url++verbatim_url+------------++Set this key to ``True`` to keep the request URL as passed to+:class:`~scrapy.Request`, without URL percent-encoding.++When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not+canonicalize the request URL, so requests whose URLs differ only in+characters that would otherwise be canonicalized get different fingerprints.++In this mode, the ``keep_fragments`` parameter is ignored, and it is+effectively true.+.. _topics-stop-response-download:diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 00042e093..7db648a45 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -138,6 +138,7 @@ class Request(object_ref):) -> None:self._encoding: str = encoding # this one has to be set firstself.method: str = str(method).upper()+ self._meta: dict[str, Any] | None = dict(meta) if meta else Noneself._set_url(url)self._set_body(body)if not isinstance(priority, int):@@ -232,7 +233,6 @@ class Request(object_ref):#: default. See :meth:`~scrapy.Spider.start`.self.dont_filter: bool = dont_filter- self._meta: dict[str, Any] | None = dict(meta) if meta else Noneself._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else Noneself._flags: list[str] | None = list(flags) if flags else None@@ -252,11 +252,17 @@ class Request(object_ref):def url(self) -> str:return self._url+ def _url_is_verbatim(self) -> bool:+ return bool(self._meta and self._meta.get("verbatim_url"))+def _set_url(self, url: str) -> None:if not isinstance(url, str):raise TypeError(f"Request url must be str, got {type(url).__name__}")- self._url = safe_url_string(url, self.encoding)+ if self._url_is_verbatim():+ self._url = url+ else:+ self._url = safe_url_string(url, self.encoding)if ("://" not in self._urldiff --git a/scrapy/utils/request.py b/scrapy/utils/request.pyindex 27d669e71..ffb7fae49 100644--- a/scrapy/utils/request.py+++ b/scrapy/utils/request.py@@ -28,7 +28,7 @@ if TYPE_CHECKING:_fingerprint_cache: WeakKeyDictionary[- Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]] = WeakKeyDictionary()@@ -71,8 +71,10 @@ def fingerprint(processed_include_headers = tuple(to_bytes(h.lower()) for h in sorted(include_headers))+ verbatim_url = bool(request.meta.get("verbatim_url"))+ effective_keep_fragments = keep_fragments and not verbatim_urlcache = _fingerprint_cache.setdefault(request, {})- cache_key = (processed_include_headers, keep_fragments)+ cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)if cache_key not in cache:# To decode bytes reliably (JSON does not support bytes), regardless of# character encoding, we use bytes.hex()@@ -84,9 +86,13 @@ def fingerprint(header_value.hex()for header_value in request.headers.getlist(header)]+ if verbatim_url:+ url = request.url+ else:+ url = canonicalize_url(request.url, keep_fragments=keep_fragments)fingerprint_data = {"method": to_unicode(request.method),- "url": canonicalize_url(request.url, keep_fragments=keep_fragments),+ "url": url,"body": (request.body or b"").hex(),"headers": headers,}@@ -108,8 +114,9 @@ class RequestFingerprinter:(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url<scrapy.Request.url>` and the values of :attr:`request.method<scrapy.Request.method>` and :attr:`request.body- <scrapy.Request.body>`. It then generates an `SHA1- <https://en.wikipedia.org/wiki/SHA-1>`_ hash.+ <scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that+ request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_+ hash."""@classmethoddiff --git a/tests/mockserver/http.py b/tests/mockserver/http.pyindex 622324a10..7ad873c02 100644--- a/tests/mockserver/http.py+++ b/tests/mockserver/http.py@@ -34,6 +34,7 @@ from .http_resources import (ResponseHeadersResource,SetCookie,Status,+ UriResource,)@@ -84,6 +85,7 @@ class Root(resource.Resource):self.putChild(b"duplicate-header", DuplicateHeaderResource())self.putChild(b"response-headers", ResponseHeadersResource())self.putChild(b"set-cookie", SetCookie())+ self.putChild(b"uri", UriResource())def getChild(self, path, request):return selfdiff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.pyindex 51da8b38f..3b60fa555 100644--- a/tests/test_downloader_handlers_http_base.py+++ b/tests/test_downloader_handlers_http_base.py@@ -803,6 +803,25 @@ class TestHttpBase(ABC):"The 'bindaddress' request meta key is not supported by" in caplog.text)+ @coroutine_test+ async def test_verbatim_url(self, mockserver: MockServer) -> None:+ # Square brackets are encoded by safe_url_string (w3lib).+ path = "/uri/items?data[0]=a"+ url = mockserver.url(path, is_secure=self.is_secure)++ # Without verbatim_url, the brackets are percent-encoded before the+ # request reaches the server.+ request = Request(url)+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == b"/uri/items?data%5B0%5D=a"++ # With verbatim_url=True the URL is sent to the server as-is.+ request = Request(url, meta={"verbatim_url": True})+ async with self.get_dh() as download_handler:+ response = await download_handler.download_request(request)+ assert response.body == path.encode()+class TestHttpsBase(TestHttpBase):is_secure = Truediff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 4c77bb1ec..fed5dbab7 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -162,6 +162,19 @@ class TestRequest:r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")assert r4.url == "http://www.example.org/r%E9sum%E9.html"+ def test_url_verbatim(self):+ r = self.request_class(+ url="http://www.scrapy.org/price/£",+ meta={"verbatim_url": True},+ )+ assert r.url == "http://www.scrapy.org/price/£"++ r = self.request_class(+ url="http://www.scrapy.org/blank space",+ meta={"verbatim_url": True},+ )+ assert r.url == "http://www.scrapy.org/blank space"+def test_body(self):r1 = self.request_class(url="http://www.example.com/")assert r1.body == b""diff --git a/tests/test_utils_request.py b/tests/test_utils_request.pyindex 70800dcec..55c46059e 100644--- a/tests/test_utils_request.py+++ b/tests/test_utils_request.py@@ -59,10 +59,14 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:class TestFingerprint:function: staticmethod[[Request], bytes] = staticmethod(fingerprint)cache: (- WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]- | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]+ WeakKeyDictionary[+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]+ ]+ | WeakKeyDictionary[+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]+ ]) = _fingerprint_cache- default_cache_key = (None, False)+ default_cache_key = (None, False, False)known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = ((Request("http://example.org"),@@ -192,6 +196,32 @@ class TestFingerprint:assert self.function(r2) != self.function(r2, keep_fragments=True)assert self.function(r1) != self.function(r2, keep_fragments=True)+ def test_verbatim_url(self):+ # verbatim_url requests skip URL canonicalization+ r1 = Request(+ "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True}+ )+ r2 = Request(+ "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True}+ )+ assert self.function(r1) != self.function(r2)++ # without verbatim_url, canonicalization makes query-param order irrelevant+ r3 = Request("http://www.example.com/query?a=1&b=2")+ r4 = Request("http://www.example.com/query?b=2&a=1")+ assert self.function(r3) == self.function(r4)++ # with verbatim_url, the fragment is always kept in the fingerprint+ r5 = Request(+ "http://www.example.com/test#fragment", meta={"verbatim_url": True}+ )+ r6 = Request("http://www.example.com/test", meta={"verbatim_url": True})+ assert self.function(r5) != self.function(r6)++ # keep_fragments parameter is ignored for verbatim_url requests+ assert self.function(r5) == self.function(r5, keep_fragments=True)+ assert self.function(r5) == self.function(r5, keep_fragments=False)+def test_method_and_body(self):r1 = Request("http://www.example.com")r2 = Request("http://www.example.com", method="POST")diff --git a/tox.ini b/tox.iniindex 5fef37e5b..cc916caea 100644--- a/tox.ini+++ b/tox.ini@@ -236,7 +236,12 @@ deps =pyOpenSSL==24.3.0queuelib==1.4.2service_identity==23.1.0- w3lib==1.20.0+ # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses+ # an inline flag placement that Python 3.11 treats as an error: global+ # flags not at the start of the expression. w3lib 1.18 stopped encoding []+ # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url+ # rely on that encoding.+ w3lib==2.1.0zope.interface==5.1.0commands =; disabling coverage