Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove remnants of w3lib < 2.6 support. (#8474)
Andrey Rakhmatullinad817afb8807
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.pyindex 6aa0e9c9a..615ec4391 100644--- a/scrapy/core/downloader/handlers/_base_streaming.py+++ b/scrapy/core/downloader/handlers/_base_streaming.py@@ -19,6 +19,8 @@ from typing import ()from urllib.parse import quote, urlsplit+from w3lib.url import add_http_if_no_scheme+from scrapy import Request, signalsfrom scrapy.exceptions import (DownloadCancelledError,@@ -35,7 +37,6 @@ from scrapy.utils._download_handlers import (normalize_bind_address,)from scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandler@@ -349,7 +350,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])proxy: str | None = request.meta.get("proxy")if not proxy:return None, None- proxy = _add_http_if_no_scheme(proxy)+ proxy = add_http_if_no_scheme(proxy)auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")return proxy, auth_header.decode("ascii") if auth_header else Nonediff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 98a85e6ca..0850759d7 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -41,6 +41,7 @@ from twisted.web.client import Response as TxResponsefrom twisted.web.http import PotentialDataLoss, _DataLossfrom twisted.web.http_headers import Headers as TxHeadersfrom twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse+from w3lib.url import add_http_if_no_schemefrom zope.interface import implementerfrom scrapy import Request, signals@@ -69,7 +70,6 @@ from scrapy.utils._ssl import _log_ssl_conn_debug_infofrom scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandler@@ -802,7 +802,7 @@ class _ScrapyAgent:bindaddress = normalize_bind_address(bindaddress)proxy = request.meta.get("proxy")if proxy:- proxy = _add_http_if_no_scheme(proxy)+ proxy = add_http_if_no_scheme(proxy)proxy_parsed = urlparse_cached(proxy)proxy_host = proxy_parsed.hostnameproxy_port = proxy_parsed.portdiff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.pyindex 31d75af75..f76e92c79 100644--- a/scrapy/utils/_deps_compat.py+++ b/scrapy/utils/_deps_compat.py@@ -5,7 +5,6 @@ from packaging.version import Versionfrom parsel import __version__ as PARSEL_VERSION_STRINGfrom twisted import version as TWISTED_VERSIONfrom twisted.python.versions import Version as TxVersion-from w3lib import __version__ as W3LIB_VERSION_STRING# improved urllib.robotparser, https://github.com/python/cpython/pull/149374STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (@@ -25,8 +24,3 @@ PARSEL_BODY_ENCODING_FIXED = PARSEL_VERSION >= Version("1.12.0")PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutablePYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")--W3LIB_VERSION = Version(W3LIB_VERSION_STRING)-# safe_url_string() drops the brackets of an IPv6 address literal in the-# netloc, https://github.com/scrapy/w3lib/pull/253-W3LIB_STRIPS_IPV6_BRACKETS = Version("2.0.0") <= W3LIB_VERSION < Version("2.4.1")diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 1292edde4..c275878b3 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -72,23 +72,15 @@ def url_has_any_extension(return any(lowercase_path.endswith(ext) for ext in extensions)-def _add_http_if_no_scheme(url: str) -> str:- if not re.match(r"^\w+://", url, flags=re.IGNORECASE):- scheme = "http:" if urlparse_cached(url).netloc else "http://"- url = scheme + url- return url--def add_http_if_no_scheme(url: str) -> str:"""Add http as the default scheme if it is missing from the url."""- if hasattr(w3lib.url, "add_http_if_no_scheme"):- warnings.warn(- "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "- "w3lib.url.add_http_if_no_scheme() instead.",- ScrapyDeprecationWarning,- stacklevel=2,- )- return _add_http_if_no_scheme(url)+ warnings.warn(+ "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "+ "w3lib.url.add_http_if_no_scheme() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return w3lib.url.add_http_if_no_scheme(url)def _is_posix_path(string: str) -> bool:@@ -138,7 +130,7 @@ def guess_scheme(url: str) -> str:http:// otherwise."""if _is_filesystem_path(url):return any_to_uri(url)- return _add_http_if_no_scheme(url)+ return w3lib.url.add_http_if_no_scheme(url)def strip_url(diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.pyindex 8181c4d17..e349a718a 100644--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py@@ -10,8 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):name = "caching_hostname_resolver_spider"async def start(self):- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})+ yield scrapy.Request("http://[::1]")if __name__ == "__main__":diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.pyindex 7cc59594b..7334db81b 100644--- a/tests/AsyncCrawlerProcess/default_name_resolver.py+++ b/tests/AsyncCrawlerProcess/default_name_resolver.py@@ -11,8 +11,7 @@ class IPv6Spider(scrapy.Spider):name = "ipv6_spider"async def start(self):- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})+ yield scrapy.Request("http://[::1]")if __name__ == "__main__":diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.pyindex f6f865e3e..2f86818d7 100644--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py@@ -10,8 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):name = "caching_hostname_resolver_spider"async def start(self):- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})+ yield scrapy.Request("http://[::1]")if __name__ == "__main__":diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.pyindex 12b894030..4c8ce8bdd 100644--- a/tests/CrawlerProcess/default_name_resolver.py+++ b/tests/CrawlerProcess/default_name_resolver.py@@ -11,8 +11,7 @@ class IPv6Spider(scrapy.Spider):name = "ipv6_spider"async def start(self):- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})+ yield scrapy.Request("http://[::1]")if __name__ == "__main__":diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.pyindex e6c593712..685d9d64a 100644--- a/tests/test_downloadermiddleware_httpcache.py+++ b/tests/test_downloadermiddleware_httpcache.py@@ -11,7 +11,6 @@ from contextlib import contextmanagerfrom pathlib import Pathfrom typing import TYPE_CHECKING, Anyfrom unittest import mock-from urllib.parse import urlparseimport pytestfrom twisted.internet.defer import Deferred@@ -1295,14 +1294,7 @@ class TestFilesystemStorageScope(ScopeTestMixin):("url", "expected"),[("http://user:[email protected]:8080/", "www.example.com"),- pytest.param(- "http://[::1]/",- "%3A%3A1",- marks=pytest.mark.skipif(- urlparse(Request("http://[::1]/").url).hostname != "::1",- reason="w3lib strips the brackets of IPv6 hosts",- ),- ),+ ("http://[::1]/", "%3A%3A1"),("file:///tmp/t.txt", "_"),],)diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.pyindex 22c499339..5a38cc516 100644--- a/tests/test_http_response_text.py+++ b/tests/test_http_response_text.py@@ -246,10 +246,7 @@ class TestTextResponse(TestResponseBase):body=b"\xef\xbb\xbfWORD\xe3\xab",)assert r6.encoding == "utf-8"- assert r6.text in {- "WORD\ufffd\ufffd", # w3lib < 1.19.0- "WORD\ufffd", # w3lib >= 1.19.0- }+ assert r6.text == "WORD\ufffd"def test_bom_is_removed_from_body(self):# Inferring encoding from body also cache decoded body as sideeffect,diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 247d38144..cb20f70c1 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -4,14 +4,12 @@ from typing import Anyfrom urllib.parse import urlparseimport pytest-import w3lib.urlfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Request, Responsefrom scrapy.linkextractors import IGNORED_EXTENSIONSfrom scrapy.spiders import Spiderfrom scrapy.utils.url import (- _add_http_if_no_scheme,_is_filesystem_path,add_http_if_no_scheme,guess_scheme,@@ -126,83 +124,13 @@ def test_url_has_any_extension(url: str, expected: bool) -> None:assert url_has_any_extension(url, deny_extensions) is expected- ("url", "expected"),- [- ("www.example.com", "http://www.example.com"),- ("example.com", "http://example.com"),- ("www.example.com/some/page.html", "http://www.example.com/some/page.html"),- ("www.example.com:80", "http://www.example.com:80"),- ("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),- ("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),- (- "username:[email protected]",- "http://username:[email protected]",- ),- (- "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- ),- ("http://www.example.com", "http://www.example.com"),- ("http://example.com", "http://example.com"),- (- "http://www.example.com/some/page.html",- "http://www.example.com/some/page.html",- ),- ("http://www.example.com:80", "http://www.example.com:80"),- (- "http://www.example.com/some/page#frag",- "http://www.example.com/some/page#frag",- ),- (- "http://www.example.com/do?a=1&b=2&c=3",- "http://www.example.com/do?a=1&b=2&c=3",- ),- (- "http://username:[email protected]",- "http://username:[email protected]",- ),- (- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- ),- ("//www.example.com", "http://www.example.com"),- ("//example.com", "http://example.com"),- ("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),- ("//www.example.com:80", "http://www.example.com:80"),- ("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),- ("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),- (- "//username:[email protected]",- "http://username:[email protected]",- ),- (- "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",- ),- ("https://www.example.com", "https://www.example.com"),- ("ftp://www.example.com", "ftp://www.example.com"),- ],-)-def test_add_http_if_no_scheme(url: str, expected: str) -> None:- assert _add_http_if_no_scheme(url) == expected---def test_add_http_if_no_scheme_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(- w3lib.url, "add_http_if_no_scheme", lambda url: url, raising=False- )+def test_add_http_if_no_scheme_deprecated() -> None:with pytest.warns(ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"):assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"-def test_add_http_if_no_scheme_not_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.delattr(w3lib.url, "add_http_if_no_scheme", raising=False)- assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"--@pytest.mark.parametrize(("url", "expected"),[diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.pyindex de609312b..b151e7bb3 100644--- a/tests/utils/bases/download_handlers_http.py+++ b/tests/utils/bases/download_handlers_http.py@@ -40,10 +40,7 @@ from scrapy.http import (Response,TextResponse,)-from scrapy.utils._deps_compat import (- TWISTED_TLS_LIMITS_OFFBY1,- W3LIB_STRIPS_IPV6_BRACKETS,-)+from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_futurefrom scrapy.utils.misc import build_from_crawlerfrom scrapy.utils.reactor import is_reactor_installed@@ -249,9 +246,6 @@ class TestHttpBase(ABC):@pytest.mark.skipif(not ipv6_loopback_available(), reason="IPv6 loopback is not available")- @pytest.mark.skipif(- W3LIB_STRIPS_IPV6_BRACKETS, reason="https://github.com/scrapy/w3lib/pull/253"- )@coroutine_testasync def test_download_ipv6_literal(self, mockserver_ipv6: MockServer) -> None:"""Download from a URL whose host is an IPv6 address literal."""@@ -2134,9 +2128,6 @@ class TestMitmProxyBase(ABC):@pytest.mark.skipif(not ipv6_loopback_available(), reason="IPv6 loopback is not available")- @pytest.mark.skipif(- W3LIB_STRIPS_IPV6_BRACKETS, reason="https://github.com/scrapy/w3lib/pull/253"- )@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)@pytest.mark.parametrize("https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]