Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Document urlparse_cached (#7777)

Adrian58ed9fdcccc9
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 7468f1b64..b83a04032 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -342,6 +342,8 @@ Other functions related to requests
.. autofunction:: scrapy.utils.request.request_from_dict
+.. autofunction:: scrapy.utils.httpobj.urlparse_cached
+
.. _topics-request-response-ref-request-callback-arguments:
diff --git a/scrapy/utils/httpobj.py b/scrapy/utils/httpobj.py
index 58b4539bf..5965e1f21 100644
--- a/scrapy/utils/httpobj.py
+++ b/scrapy/utils/httpobj.py
@@ -16,8 +16,15 @@ _urlparse_cache: WeakKeyDictionary[Request | Response, ParseResult] = (
def urlparse_cached(request_or_response: Request | Response) -> ParseResult:
- """Return urlparse.urlparse caching the result, where the argument can be a
- Request or Response object
+ """Return the result of parsing the URL of *request_or_response*, a
+ :class:`~scrapy.Request` or :class:`~scrapy.http.Response` object, with
+ :func:`urllib.parse.urlparse`.
+
+ The result is cached, using a :class:`weakref.WeakKeyDictionary` keyed on
+ *request_or_response*, so that the URL of a given object is parsed only
+ once. Prefer this function over calling :func:`urllib.parse.urlparse` on
+ ``request_or_response.url`` directly when the same URL may be parsed more
+ than once.
"""
if request_or_response not in _urlparse_cache:
_urlparse_cache[request_or_response] = urlparse(request_or_response.url)