Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Support sending requests with "unsafe" URLs (#7473)

Adrianb2d8b06be61d
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index d1e4851d9..a4f031803 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -117,6 +117,9 @@ Request objects
:param encoding: the encoding of this request (defaults to ``'utf-8'``).
This encoding will be used to percent-encode the URL and to convert the
body to bytes (if given as a string).
+
+ To disable URL percent-encoding for a request, use the
+ :reqmeta:`verbatim_url` request meta key.
:type encoding: str
:param priority: sets :attr:`priority`, defaults to ``0``.
@@ -136,9 +139,13 @@ Request objects
.. attribute:: Request.url
- A string containing the URL of this request. Keep in mind that this
- attribute contains the escaped URL, so it can differ from the URL passed in
- the ``__init__()`` method.
+ A string containing the URL of this request.
+
+ Keep in mind that this attribute contains the escaped URL, so it can
+ differ from the URL passed in the ``__init__()`` method.
+
+ If :reqmeta:`verbatim_url` is set to ``True``, the URL is kept as
+ passed to ``__init__()``.
This attribute is read-only. To change the URL of a Request use
:meth:`replace`.
@@ -541,6 +548,11 @@ in your :meth:`fingerprint` method implementation:
.. autofunction:: scrapy.utils.request.fingerprint
+By default, request fingerprinting canonicalizes the request URL. If
+:reqmeta:`verbatim_url` is set to ``True``, fingerprinting does not
+canonicalize the URL, and the ``keep_fragments`` parameter is ignored (it is
+effectively true).
+
For example, to take the value of a request header named ``X-ID`` into
account:
@@ -710,6 +722,7 @@ Those are:
* :reqmeta:`redirect_reasons`
* :reqmeta:`redirect_urls`
* :reqmeta:`referrer_policy`
+* :reqmeta:`verbatim_url`
.. reqmeta:: bindaddress
@@ -786,6 +799,21 @@ The meta key is used set retry times per request. When initialized, the
:reqmeta:`max_retry_times` meta key takes higher precedence over the
:setting:`RETRY_TIMES` setting.
+.. reqmeta:: verbatim_url
+
+verbatim_url
+------------
+
+Set this key to ``True`` to keep the request URL as passed to
+:class:`~scrapy.Request`, without URL percent-encoding.
+
+When this key is enabled, :func:`~scrapy.utils.request.fingerprint` does not
+canonicalize the request URL, so requests whose URLs differ only in
+characters that would otherwise be canonicalized get different fingerprints.
+
+In this mode, the ``keep_fragments`` parameter is ignored, and it is
+effectively true.
+
.. _topics-stop-response-download:
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.py
index 00042e093..7db648a45 100644
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@ -138,6 +138,7 @@ class Request(object_ref):
) -> None:
self._encoding: str = encoding # this one has to be set first
self.method: str = str(method).upper()
+ self._meta: dict[str, Any] | None = dict(meta) if meta else None
self._set_url(url)
self._set_body(body)
if not isinstance(priority, int):
@@ -232,7 +233,6 @@ class Request(object_ref):
#: default. See :meth:`~scrapy.Spider.start`.
self.dont_filter: bool = dont_filter
- self._meta: dict[str, Any] | None = dict(meta) if meta else None
self._cb_kwargs: dict[str, Any] | None = dict(cb_kwargs) if cb_kwargs else None
self._flags: list[str] | None = list(flags) if flags else None
@@ -252,11 +252,17 @@ class Request(object_ref):
def url(self) -> str:
return self._url
+ def _url_is_verbatim(self) -> bool:
+ return bool(self._meta and self._meta.get("verbatim_url"))
+
def _set_url(self, url: str) -> None:
if not isinstance(url, str):
raise TypeError(f"Request url must be str, got {type(url).__name__}")
- self._url = safe_url_string(url, self.encoding)
+ if self._url_is_verbatim():
+ self._url = url
+ else:
+ self._url = safe_url_string(url, self.encoding)
if (
"://" not in self._url
diff --git a/scrapy/utils/request.py b/scrapy/utils/request.py
index 27d669e71..ffb7fae49 100644
--- a/scrapy/utils/request.py
+++ b/scrapy/utils/request.py
@@ -28,7 +28,7 @@ if TYPE_CHECKING:
_fingerprint_cache: WeakKeyDictionary[
- Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]
+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
] = WeakKeyDictionary()
@@ -71,8 +71,10 @@ def fingerprint(
processed_include_headers = tuple(
to_bytes(h.lower()) for h in sorted(include_headers)
)
+ verbatim_url = bool(request.meta.get("verbatim_url"))
+ effective_keep_fragments = keep_fragments and not verbatim_url
cache = _fingerprint_cache.setdefault(request, {})
- cache_key = (processed_include_headers, keep_fragments)
+ cache_key = (processed_include_headers, effective_keep_fragments, verbatim_url)
if cache_key not in cache:
# To decode bytes reliably (JSON does not support bytes), regardless of
# character encoding, we use bytes.hex()
@@ -84,9 +86,13 @@ def fingerprint(
header_value.hex()
for header_value in request.headers.getlist(header)
]
+ if verbatim_url:
+ url = request.url
+ else:
+ url = canonicalize_url(request.url, keep_fragments=keep_fragments)
fingerprint_data = {
"method": to_unicode(request.method),
- "url": canonicalize_url(request.url, keep_fragments=keep_fragments),
+ "url": url,
"body": (request.body or b"").hex(),
"headers": headers,
}
@@ -108,8 +114,9 @@ class RequestFingerprinter:
(:func:`w3lib.url.canonicalize_url`) of :attr:`request.url
<scrapy.Request.url>` and the values of :attr:`request.method
<scrapy.Request.method>` and :attr:`request.body
- <scrapy.Request.body>`. It then generates an `SHA1
- <https://en.wikipedia.org/wiki/SHA-1>`_ hash.
+ <scrapy.Request.body>`, unless :reqmeta:`verbatim_url` is true for that
+ request. It then generates an `SHA1 <https://en.wikipedia.org/wiki/SHA-1>`_
+ hash.
"""
@classmethod
diff --git a/tests/mockserver/http.py b/tests/mockserver/http.py
index 622324a10..7ad873c02 100644
--- a/tests/mockserver/http.py
+++ b/tests/mockserver/http.py
@@ -34,6 +34,7 @@ from .http_resources import (
ResponseHeadersResource,
SetCookie,
Status,
+ UriResource,
)
@@ -84,6 +85,7 @@ class Root(resource.Resource):
self.putChild(b"duplicate-header", DuplicateHeaderResource())
self.putChild(b"response-headers", ResponseHeadersResource())
self.putChild(b"set-cookie", SetCookie())
+ self.putChild(b"uri", UriResource())
def getChild(self, path, request):
return self
diff --git a/tests/test_downloader_handlers_http_base.py b/tests/test_downloader_handlers_http_base.py
index 51da8b38f..3b60fa555 100644
--- a/tests/test_downloader_handlers_http_base.py
+++ b/tests/test_downloader_handlers_http_base.py
@@ -803,6 +803,25 @@ class TestHttpBase(ABC):
"The 'bindaddress' request meta key is not supported by" in caplog.text
)
+ @coroutine_test
+ async def test_verbatim_url(self, mockserver: MockServer) -> None:
+ # Square brackets are encoded by safe_url_string (w3lib).
+ path = "/uri/items?data[0]=a"
+ url = mockserver.url(path, is_secure=self.is_secure)
+
+ # Without verbatim_url, the brackets are percent-encoded before the
+ # request reaches the server.
+ request = Request(url)
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == b"/uri/items?data%5B0%5D=a"
+
+ # With verbatim_url=True the URL is sent to the server as-is.
+ request = Request(url, meta={"verbatim_url": True})
+ async with self.get_dh() as download_handler:
+ response = await download_handler.download_request(request)
+ assert response.body == path.encode()
+
class TestHttpsBase(TestHttpBase):
is_secure = True
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index 4c77bb1ec..fed5dbab7 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -162,6 +162,19 @@ class TestRequest:
r4 = self.request_class(url="http://www.example.org/r%E9sum%E9.html")
assert r4.url == "http://www.example.org/r%E9sum%E9.html"
+ def test_url_verbatim(self):
+ r = self.request_class(
+ url="http://www.scrapy.org/price/£",
+ meta={"verbatim_url": True},
+ )
+ assert r.url == "http://www.scrapy.org/price/£"
+
+ r = self.request_class(
+ url="http://www.scrapy.org/blank space",
+ meta={"verbatim_url": True},
+ )
+ assert r.url == "http://www.scrapy.org/blank space"
+
def test_body(self):
r1 = self.request_class(url="http://www.example.com/")
assert r1.body == b""
diff --git a/tests/test_utils_request.py b/tests/test_utils_request.py
index 70800dcec..55c46059e 100644
--- a/tests/test_utils_request.py
+++ b/tests/test_utils_request.py
@@ -59,10 +59,14 @@ def test_request_httprepr_for_non_http_request(r: Request) -> None:
class TestFingerprint:
function: staticmethod[[Request], bytes] = staticmethod(fingerprint)
cache: (
- WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], bytes]]
- | WeakKeyDictionary[Request, dict[tuple[tuple[bytes, ...] | None, bool], str]]
+ WeakKeyDictionary[
+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], bytes]
+ ]
+ | WeakKeyDictionary[
+ Request, dict[tuple[tuple[bytes, ...] | None, bool, bool], str]
+ ]
) = _fingerprint_cache
- default_cache_key = (None, False)
+ default_cache_key = (None, False, False)
known_hashes: tuple[tuple[Request, bytes | str, dict[str, Any]], ...] = (
(
Request("http://example.org"),
@@ -192,6 +196,32 @@ class TestFingerprint:
assert self.function(r2) != self.function(r2, keep_fragments=True)
assert self.function(r1) != self.function(r2, keep_fragments=True)
+ def test_verbatim_url(self):
+ # verbatim_url requests skip URL canonicalization
+ r1 = Request(
+ "http://www.example.com/query?a=1&b=2", meta={"verbatim_url": True}
+ )
+ r2 = Request(
+ "http://www.example.com/query?b=2&a=1", meta={"verbatim_url": True}
+ )
+ assert self.function(r1) != self.function(r2)
+
+ # without verbatim_url, canonicalization makes query-param order irrelevant
+ r3 = Request("http://www.example.com/query?a=1&b=2")
+ r4 = Request("http://www.example.com/query?b=2&a=1")
+ assert self.function(r3) == self.function(r4)
+
+ # with verbatim_url, the fragment is always kept in the fingerprint
+ r5 = Request(
+ "http://www.example.com/test#fragment", meta={"verbatim_url": True}
+ )
+ r6 = Request("http://www.example.com/test", meta={"verbatim_url": True})
+ assert self.function(r5) != self.function(r6)
+
+ # keep_fragments parameter is ignored for verbatim_url requests
+ assert self.function(r5) == self.function(r5, keep_fragments=True)
+ assert self.function(r5) == self.function(r5, keep_fragments=False)
+
def test_method_and_body(self):
r1 = Request("http://www.example.com")
r2 = Request("http://www.example.com", method="POST")
diff --git a/tox.ini b/tox.ini
index 5fef37e5b..cc916caea 100644
--- a/tox.ini
+++ b/tox.ini
@@ -236,7 +236,12 @@ deps =
pyOpenSSL==24.3.0
queuelib==1.4.2
service_identity==23.1.0
- w3lib==1.20.0
+ # w3lib 1.17 fails to import on PyPy 3.11 because its encoding regex uses
+ # an inline flag placement that Python 3.11 treats as an error: global
+ # flags not at the start of the expression. w3lib 1.18 stopped encoding []
+ # in URLs until 2.1.0 brought that behavior back. Tests for verbatim_url
+ # rely on that encoding.
+ w3lib==2.1.0
zope.interface==5.1.0
commands =
; disabling coverage