Scrapy, a fast high-level web crawling & scraping framework for Python.
Replace urlparse with urlparse_cached where possible (#6229)
Laerte Pereiraee1189512f65
diff --git a/docs/topics/media-pipeline.rst b/docs/topics/media-pipeline.rstindex da0587aa4..c96dd0f99 100644--- a/docs/topics/media-pipeline.rst+++ b/docs/topics/media-pipeline.rst@@ -532,14 +532,14 @@ See here the methods that you can override in your custom Files Pipeline:.. code-block:: pythonfrom pathlib import PurePosixPath- from urllib.parse import urlparse+ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.pipelines.files import FilesPipelineclass MyFilesPipeline(FilesPipeline):def file_path(self, request, response=None, info=None, *, item=None):- return "files/" + PurePosixPath(urlparse(request.url).path).name+ return "files/" + PurePosixPath(urlparse_cached(request).path).nameSimilarly, you can use the ``item`` to determine the file path based on some itemproperty.@@ -690,14 +690,14 @@ See here the methods that you can override in your custom Images Pipeline:.. code-block:: pythonfrom pathlib import PurePosixPath- from urllib.parse import urlparse+ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.pipelines.images import ImagesPipelineclass MyImagesPipeline(ImagesPipeline):def file_path(self, request, response=None, info=None, *, item=None):- return "files/" + PurePosixPath(urlparse(request.url).path).name+ return "files/" + PurePosixPath(urlparse_cached(request).path).nameSimilarly, you can use the ``item`` to determine the file path based on some itemproperty.diff --git a/scrapy/core/http2/stream.py b/scrapy/core/http2/stream.pyindex 39d5921f4..0f282d83d 100644--- a/scrapy/core/http2/stream.py+++ b/scrapy/core/http2/stream.py@@ -2,7 +2,6 @@ import loggingfrom enum import Enumfrom io import BytesIOfrom typing import TYPE_CHECKING, Dict, List, Optional, Tuple-from urllib.parse import urlparsefrom h2.errors import ErrorCodesfrom h2.exceptions import H2Error, ProtocolError, StreamClosedError@@ -15,6 +14,7 @@ from twisted.web.client import ResponseFailedfrom scrapy.http import Requestfrom scrapy.http.headers import Headersfrom scrapy.responsetypes import responsetypes+from scrapy.utils.httpobj import urlparse_cachedif TYPE_CHECKING:from scrapy.core.http2.protocol import H2ClientProtocol@@ -185,7 +185,7 @@ class Stream:def check_request_url(self) -> bool:# Make sure that we are sending the request to the correct URL- url = urlparse(self._request.url)+ url = urlparse_cached(self._request)return (url.netloc == str(self._protocol.metadata["uri"].host, "utf-8")or url.netloc == str(self._protocol.metadata["uri"].netloc, "utf-8")@@ -194,7 +194,7 @@ class Stream:)def _get_request_headers(self) -> List[Tuple[str, str]]:- url = urlparse(self._request.url)+ url = urlparse_cached(self._request)path = url.pathif url.query:diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 83afdf7d7..24089afea 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -2,7 +2,7 @@ from __future__ import annotationsimport loggingfrom typing import TYPE_CHECKING, Any, List, Union, cast-from urllib.parse import urljoin, urlparse+from urllib.parse import urljoinfrom w3lib.url import safe_url_string@@ -125,7 +125,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):assert response.headers["Location"] is not Nonelocation = safe_url_string(response.headers["Location"])if response.headers["Location"].startswith(b"//"):- request_scheme = urlparse(request.url).scheme+ request_scheme = urlparse_cached(request).schemelocation = request_scheme + "://" + location.lstrip("/")redirected_url = urljoin(request.url, location)diff --git a/tests/CrawlerRunner/ip_address.py b/tests/CrawlerRunner/ip_address.pyindex 23260ab0d..5bf7512bc 100644--- a/tests/CrawlerRunner/ip_address.py+++ b/tests/CrawlerRunner/ip_address.py@@ -9,6 +9,7 @@ from twisted.python.runtime import platformfrom scrapy import Request, Spiderfrom scrapy.crawler import CrawlerRunner+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.log import configure_loggingfrom tests.mockserver import MockDNSServer, MockServer@@ -30,7 +31,7 @@ class LocalhostSpider(Spider):yield Request(self.url)def parse(self, response):- netloc = urlparse(response.url).netloc+ netloc = urlparse_cached(response).netlochost = netloc.split(":")[0]self.logger.info(f"Host: {host}")self.logger.info(f"Type: {type(response.ip_address)}")diff --git a/tests/test_http_cookies.py b/tests/test_http_cookies.pyindex 9e43b72b0..8b5554914 100644--- a/tests/test_http_cookies.py+++ b/tests/test_http_cookies.py@@ -1,8 +1,8 @@from unittest import TestCase-from urllib.parse import urlparsefrom scrapy.http import Request, Responsefrom scrapy.http.cookies import WrappedRequest, WrappedResponse+from scrapy.utils.httpobj import urlparse_cachedclass WrappedRequestTest(TestCase):@@ -17,12 +17,12 @@ class WrappedRequestTest(TestCase):self.assertEqual(self.wrapped.full_url, self.request.url)def test_get_host(self):- self.assertEqual(self.wrapped.get_host(), urlparse(self.request.url).netloc)- self.assertEqual(self.wrapped.host, urlparse(self.request.url).netloc)+ self.assertEqual(self.wrapped.get_host(), urlparse_cached(self.request).netloc)+ self.assertEqual(self.wrapped.host, urlparse_cached(self.request).netloc)def test_get_type(self):- self.assertEqual(self.wrapped.get_type(), urlparse(self.request.url).scheme)- self.assertEqual(self.wrapped.type, urlparse(self.request.url).scheme)+ self.assertEqual(self.wrapped.get_type(), urlparse_cached(self.request).scheme)+ self.assertEqual(self.wrapped.type, urlparse_cached(self.request).scheme)def test_is_unverifiable(self):self.assertFalse(self.wrapped.is_unverifiable())diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 6dc9ec8b7..04fcaa231 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -5,7 +5,7 @@ import warningsimport xmlrpc.clientfrom typing import Any, Dict, Listfrom unittest import mock-from urllib.parse import parse_qs, unquote_to_bytes, urlparse+from urllib.parse import parse_qs, unquote_to_bytesfrom scrapy.http import (FormRequest,@@ -16,6 +16,7 @@ from scrapy.http import (XmlRpcRequest,)from scrapy.http.request import NO_CALLBACK+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode@@ -617,8 +618,8 @@ class FormRequestTest(RequestTest):method="GET",formdata=(("foo", "bar"), ("foo", "baz")),)- self.assertEqual(urlparse(req.url).hostname, "www.example.com")- self.assertEqual(urlparse(req.url).query, "foo=bar&foo=baz")+ self.assertEqual(urlparse_cached(req).hostname, "www.example.com")+ self.assertEqual(urlparse_cached(req).query, "foo=bar&foo=baz")def test_from_response_override_duplicate_form_key(self):response = _buildresponse(@@ -666,8 +667,8 @@ class FormRequestTest(RequestTest):response, formdata={"one": ["two", "three"], "six": "seven"})self.assertEqual(r1.method, "GET")- self.assertEqual(urlparse(r1.url).hostname, "www.example.com")- self.assertEqual(urlparse(r1.url).path, "/this/get.php")+ self.assertEqual(urlparse_cached(r1).hostname, "www.example.com")+ self.assertEqual(urlparse_cached(r1).path, "/this/get.php")fs = _qs(r1)self.assertEqual(set(fs[b"test"]), {b"val1", b"val2"})self.assertEqual(set(fs[b"one"]), {b"two", b"three"})diff --git a/tests/test_scheduler_base.py b/tests/test_scheduler_base.pyindex 76ca777a8..5db2e4e50 100644--- a/tests/test_scheduler_base.py+++ b/tests/test_scheduler_base.py@@ -1,6 +1,6 @@from typing import Dict, Optionalfrom unittest import TestCase-from urllib.parse import urljoin, urlparse+from urllib.parse import urljoinfrom testfixtures import LogCapturefrom twisted.internet import defer@@ -9,6 +9,7 @@ from twisted.trial.unittest import TestCase as TwistedTestCasefrom scrapy.core.scheduler import BaseSchedulerfrom scrapy.http import Requestfrom scrapy.spiders import Spider+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.request import fingerprintfrom scrapy.utils.test import get_crawlerfrom tests.mockserver import MockServer@@ -57,7 +58,7 @@ class TestSpider(Spider):self.start_urls = map(mockserver.url, PATHS)def parse(self, response):- return {"path": urlparse(response.url).path}+ return {"path": urlparse_cached(response).path}class InterfaceCheckMixin: