Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove scrapy.core.downloader.webclient._parse().
Andrey Rakhmatullin0a80871c3a2b
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex aa8a1a2a4..74a6e54ee 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -9,7 +9,7 @@ from contextlib import suppressfrom io import BytesIOfrom time import timefrom typing import TYPE_CHECKING, Any, TypedDict, TypeVar-from urllib.parse import urldefrag, urlunparse+from urllib.parse import urldefrag, urlparsefrom twisted.internet import sslfrom twisted.internet.defer import CancelledError, Deferred, succeed@@ -32,11 +32,12 @@ from zope.interface import implementerfrom scrapy import Request, Spider, signalsfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settings-from scrapy.core.downloader.webclient import _parsefrom scrapy.exceptions import StopDownloadfrom scrapy.http import Headers, Responsefrom scrapy.responsetypes import responsetypes+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode+from scrapy.utils.url import add_http_if_no_schemeif TYPE_CHECKING:from twisted.internet.base import ReactorBase@@ -378,12 +379,15 @@ class ScrapyAgent:bindaddress = request.meta.get("bindaddress") or self._bindAddressproxy = request.meta.get("proxy")if proxy:- proxyScheme, proxyNetloc, proxyHost, proxyPort, proxyParams = _parse(proxy)- scheme = _parse(request.url)[0]- proxyHost_str = to_unicode(proxyHost)- if scheme == b"https":+ proxy = add_http_if_no_scheme(proxy)+ proxy_parsed = urlparse(proxy)+ proxy_host = proxy_parsed.hostname+ proxy_port = proxy_parsed.port+ if not proxy_port:+ proxy_port = 443 if proxy_parsed.scheme == "https" else 80+ if urlparse_cached(request).scheme == "https":proxyAuth = request.headers.get(b"Proxy-Authorization", None)- proxyConf = (proxyHost_str, proxyPort, proxyAuth)+ proxyConf = (proxy_host, proxy_port, proxyAuth)return self._TunnelingAgent(reactor=reactor,proxyConf=proxyConf,@@ -392,13 +396,9 @@ class ScrapyAgent:bindAddress=bindaddress,pool=self._pool,)- proxyScheme = proxyScheme or b"http"- proxyURI = urlunparse(- (proxyScheme, proxyNetloc, proxyParams, b"", b"", b"")- )return self._ProxyAgent(reactor=reactor,- proxyURI=to_bytes(proxyURI, encoding="ascii"),+ proxyURI=to_bytes(proxy, encoding="ascii"),connectTimeout=timeout,bindAddress=bindaddress,pool=self._pool,diff --git a/scrapy/core/downloader/handlers/http2.py b/scrapy/core/downloader/handlers/http2.pyindex f0f9ceeb7..d0a95ee9d 100644--- a/scrapy/core/downloader/handlers/http2.py+++ b/scrapy/core/downloader/handlers/http2.py@@ -8,8 +8,8 @@ from twisted.internet.error import TimeoutErrorfrom twisted.web.client import URIfrom scrapy.core.downloader.contextfactory import load_context_factory_from_settings-from scrapy.core.downloader.webclient import _parsefrom scrapy.core.http2.agent import H2Agent, H2ConnectionPool, ScrapyProxyH2Agent+from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytesif TYPE_CHECKING:@@ -75,10 +75,7 @@ class ScrapyH2Agent:bind_address = request.meta.get("bindaddress") or self._bind_addressproxy = request.meta.get("proxy")if proxy:- _, _, proxy_host, proxy_port, proxy_params = _parse(proxy)- scheme = _parse(request.url)[0]-- if scheme == b"https":+ if urlparse_cached(request).scheme == "https":# ToDoraise NotImplementedError("Tunneling via CONNECT method using HTTP/2.0 is not yet supported"diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 09751ea1a..e5c2255af 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -2,11 +2,10 @@from __future__ import annotations-import reimport warningsfrom time import timefrom typing import TYPE_CHECKING-from urllib.parse import ParseResult, urldefrag, urlparse, urlunparse+from urllib.parse import urldefrag, urlparse, urlunparsefrom twisted.internet import deferfrom twisted.internet.protocol import ClientFactory@@ -22,34 +21,6 @@ if TYPE_CHECKING:from scrapy import Request-def _parsed_url_args(parsed: ParseResult) -> tuple[bytes, bytes, bytes, int, bytes]:- # Assume parsed is urlparse-d from Request.url,- # which was passed via safe_url_string and is ascii-only.- path_str = urlunparse(("", "", parsed.path or "/", parsed.params, parsed.query, ""))- path = to_bytes(path_str, encoding="ascii")- assert parsed.hostname is not None- host = to_bytes(parsed.hostname, encoding="ascii")- port = parsed.port- scheme = to_bytes(parsed.scheme, encoding="ascii")- netloc = to_bytes(parsed.netloc, encoding="ascii")- if port is None:- port = 443 if scheme == b"https" else 80- return scheme, netloc, host, port, path---def _parse(url: str) -> tuple[bytes, bytes, bytes, int, bytes]:- """Return tuple of (scheme, netloc, host, port, path),- all in bytes except for port which is int.- Assume url is from Request.url, which was passed via safe_url_string- and is ascii-only.- """- url = url.strip()- if not re.match(r"^\w+://", url):- url = "//" + url- parsed = urlparse(url)- return _parsed_url_args(parsed)--class ScrapyHTTPPageGetter(HTTPClient):delimiter = b"\n"@@ -142,14 +113,29 @@ class ScrapyHTTPClientFactory(ClientFactory):)def _set_connection_attributes(self, request):- parsed = urlparse_cached(request)- self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(- parsed- )proxy = request.meta.get("proxy")if proxy:- self.scheme, _, self.host, self.port, _ = _parse(proxy)+ proxy_parsed = urlparse(to_bytes(proxy, encoding="ascii"))+ self.scheme = proxy_parsed.scheme+ self.host = proxy_parsed.hostname+ self.port = proxy_parsed.port+ self.netloc = proxy_parsed.netloc+ if self.port is None:+ self.port = 443 if proxy_parsed.scheme == b"https" else 80self.path = self.url+ else:+ parsed = urlparse_cached(request)+ path_str = urlunparse(+ ("", "", parsed.path or "/", parsed.params, parsed.query, "")+ )+ self.path = to_bytes(path_str, encoding="ascii")+ assert parsed.hostname is not None+ self.host = to_bytes(parsed.hostname, encoding="ascii")+ self.port = parsed.port+ self.scheme = to_bytes(parsed.scheme, encoding="ascii")+ self.netloc = to_bytes(parsed.netloc, encoding="ascii")+ if self.port is None:+ self.port = 443 if self.scheme == b"https" else 80def __init__(self, request: Request, timeout: float = 180):warnings.warn(diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 64f615bfe..ae2030fe6 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -785,9 +785,6 @@ class HttpProxyTestCase(unittest.TestCase):class Http10ProxyTestCase(HttpProxyTestCase):download_handler_cls: type = HTTP10DownloadHandler- def test_download_with_proxy_https_noconnect(self):- raise unittest.SkipTest("noconnect is not supported in HTTP10DownloadHandler")-class Http11ProxyTestCase(HttpProxyTestCase):download_handler_cls: type = HTTP11DownloadHandlerdiff --git a/tests/test_webclient.py b/tests/test_webclient.pyindex fa19b350b..1b4ad2f2f 100644--- a/tests/test_webclient.py+++ b/tests/test_webclient.py@@ -8,6 +8,7 @@ from __future__ import annotationsimport shutilfrom pathlib import Pathfrom tempfile import mkdtemp+from urllib.parse import urlparseimport OpenSSL.SSLimport pytest@@ -61,72 +62,6 @@ def getPage(url, contextFactory=None, response_transform=None, *args, **kwargs):).deferred-class ParseUrlTestCase(unittest.TestCase):- """Test URL parsing facility and defaults values."""-- def _parse(self, url):- f = client.ScrapyHTTPClientFactory(Request(url))- return (f.scheme, f.netloc, f.host, f.port, f.path)-- def testParse(self):- lip = "127.0.0.1"- tests = (- (- "http://127.0.0.1?c=v&c2=v2#fragment",- ("http", lip, lip, 80, "/?c=v&c2=v2"),- ),- (- "http://127.0.0.1/?c=v&c2=v2#fragment",- ("http", lip, lip, 80, "/?c=v&c2=v2"),- ),- (- "http://127.0.0.1/foo?c=v&c2=v2#frag",- ("http", lip, lip, 80, "/foo?c=v&c2=v2"),- ),- (- "http://127.0.0.1:100?c=v&c2=v2#fragment",- ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"),- ),- (- "http://127.0.0.1:100/?c=v&c2=v2#frag",- ("http", lip + ":100", lip, 100, "/?c=v&c2=v2"),- ),- (- "http://127.0.0.1:100/foo?c=v&c2=v2#frag",- ("http", lip + ":100", lip, 100, "/foo?c=v&c2=v2"),- ),- ("http://127.0.0.1", ("http", lip, lip, 80, "/")),- ("http://127.0.0.1/", ("http", lip, lip, 80, "/")),- ("http://127.0.0.1/foo", ("http", lip, lip, 80, "/foo")),- ("http://127.0.0.1?param=value", ("http", lip, lip, 80, "/?param=value")),- ("http://127.0.0.1/?param=value", ("http", lip, lip, 80, "/?param=value")),- (- "http://127.0.0.1:12345/foo",- ("http", lip + ":12345", lip, 12345, "/foo"),- ),- ("http://spam:12345/foo", ("http", "spam:12345", "spam", 12345, "/foo")),- (- "http://spam.test.org/foo",- ("http", "spam.test.org", "spam.test.org", 80, "/foo"),- ),- ("https://127.0.0.1/foo", ("https", lip, lip, 443, "/foo")),- (- "https://127.0.0.1/?param=value",- ("https", lip, lip, 443, "/?param=value"),- ),- ("https://127.0.0.1:12345/", ("https", lip + ":12345", lip, 12345, "/")),- (- "http://scrapytest.org/foo ",- ("http", "scrapytest.org", "scrapytest.org", 80, "/foo"),- ),- ("http://egg:7890 ", ("http", "egg:7890", "egg", 7890, "/")),- )-- for url, test in tests:- test = tuple(to_bytes(x) if not isinstance(x, int) else x for x in test)- self.assertEqual(client._parse(url), test, url)--@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")class ScrapyHTTPPageGetterTests(unittest.TestCase):def test_earlyHeaders(self):@@ -388,9 +323,9 @@ class WebClientTestCase(unittest.TestCase):def testFactoryInfo(self):url = self.getURL("file")- _, _, host, port, _ = client._parse(url)+ parsed = urlparse(url)factory = client.ScrapyHTTPClientFactory(Request(url))- reactor.connectTCP(to_unicode(host), port, factory)+ reactor.connectTCP(parsed.hostname, parsed.port, factory)return factory.deferred.addCallback(self._cbFactoryInfo, factory)def _cbFactoryInfo(self, ignoredResult, factory):