Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove remnants of w3lib < 2.6 support. (#8474)

Andrey Rakhmatullinad817afb8807
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 6aa0e9c9a..615ec4391 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -19,6 +19,8 @@ from typing import (
)
from urllib.parse import quote, urlsplit
+from w3lib.url import add_http_if_no_scheme
+
from scrapy import Request, signals
from scrapy.exceptions import (
DownloadCancelledError,
@@ -35,7 +37,6 @@ from scrapy.utils._download_handlers import (
normalize_bind_address,
)
from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
@@ -349,7 +350,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, Generic[_ResponseT])
proxy: str | None = request.meta.get("proxy")
if not proxy:
return None, None
- proxy = _add_http_if_no_scheme(proxy)
+ proxy = add_http_if_no_scheme(proxy)
auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
return proxy, auth_header.decode("ascii") if auth_header else None
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 98a85e6ca..0850759d7 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -41,6 +41,7 @@ from twisted.web.client import Response as TxResponse
from twisted.web.http import PotentialDataLoss, _DataLoss
from twisted.web.http_headers import Headers as TxHeaders
from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer, IResponse
+from w3lib.url import add_http_if_no_scheme
from zope.interface import implementer
from scrapy import Request, signals
@@ -69,7 +70,6 @@ from scrapy.utils._ssl import _log_ssl_conn_debug_info
from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
-from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
@@ -802,7 +802,7 @@ class _ScrapyAgent:
bindaddress = normalize_bind_address(bindaddress)
proxy = request.meta.get("proxy")
if proxy:
- proxy = _add_http_if_no_scheme(proxy)
+ proxy = add_http_if_no_scheme(proxy)
proxy_parsed = urlparse_cached(proxy)
proxy_host = proxy_parsed.hostname
proxy_port = proxy_parsed.port
diff --git a/scrapy/utils/_deps_compat.py b/scrapy/utils/_deps_compat.py
index 31d75af75..f76e92c79 100644
--- a/scrapy/utils/_deps_compat.py
+++ b/scrapy/utils/_deps_compat.py
@@ -5,7 +5,6 @@ from packaging.version import Version
from parsel import __version__ as PARSEL_VERSION_STRING
from twisted import version as TWISTED_VERSION
from twisted.python.versions import Version as TxVersion
-from w3lib import __version__ as W3LIB_VERSION_STRING
# improved urllib.robotparser, https://github.com/python/cpython/pull/149374
STDLIB_IMPROVED_ROBOTFILEPARSER = sys.version_info >= (3, 14, 5) or (
@@ -25,8 +24,3 @@ PARSEL_BODY_ENCODING_FIXED = PARSEL_VERSION >= Version("1.12.0")
PYOPENSSL_VERSION = Version(PYOPENSSL_VERSION_STRING)
# SSL.Context.set_cipher_list() creates a temporary connection, making the context immutable
PYOPENSSL_SET_CIPHER_LIST_TMP_CONN = PYOPENSSL_VERSION < Version("25.2.0")
-
-W3LIB_VERSION = Version(W3LIB_VERSION_STRING)
-# safe_url_string() drops the brackets of an IPv6 address literal in the
-# netloc, https://github.com/scrapy/w3lib/pull/253
-W3LIB_STRIPS_IPV6_BRACKETS = Version("2.0.0") <= W3LIB_VERSION < Version("2.4.1")
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1292edde4..c275878b3 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -72,23 +72,15 @@ def url_has_any_extension(
return any(lowercase_path.endswith(ext) for ext in extensions)
-def _add_http_if_no_scheme(url: str) -> str:
- if not re.match(r"^\w+://", url, flags=re.IGNORECASE):
- scheme = "http:" if urlparse_cached(url).netloc else "http://"
- url = scheme + url
- return url
-
-
def add_http_if_no_scheme(url: str) -> str:
"""Add http as the default scheme if it is missing from the url."""
- if hasattr(w3lib.url, "add_http_if_no_scheme"):
- warnings.warn(
- "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "
- "w3lib.url.add_http_if_no_scheme() instead.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return _add_http_if_no_scheme(url)
+ warnings.warn(
+ "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "
+ "w3lib.url.add_http_if_no_scheme() instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return w3lib.url.add_http_if_no_scheme(url)
def _is_posix_path(string: str) -> bool:
@@ -138,7 +130,7 @@ def guess_scheme(url: str) -> str:
http:// otherwise."""
if _is_filesystem_path(url):
return any_to_uri(url)
- return _add_http_if_no_scheme(url)
+ return w3lib.url.add_http_if_no_scheme(url)
def strip_url(
diff --git a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
index 8181c4d17..e349a718a 100644
--- a/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/AsyncCrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -10,8 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
name = "caching_hostname_resolver_spider"
async def start(self):
- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
+ yield scrapy.Request("http://[::1]")
if __name__ == "__main__":
diff --git a/tests/AsyncCrawlerProcess/default_name_resolver.py b/tests/AsyncCrawlerProcess/default_name_resolver.py
index 7cc59594b..7334db81b 100644
--- a/tests/AsyncCrawlerProcess/default_name_resolver.py
+++ b/tests/AsyncCrawlerProcess/default_name_resolver.py
@@ -11,8 +11,7 @@ class IPv6Spider(scrapy.Spider):
name = "ipv6_spider"
async def start(self):
- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
+ yield scrapy.Request("http://[::1]")
if __name__ == "__main__":
diff --git a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
index f6f865e3e..2f86818d7 100644
--- a/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
+++ b/tests/CrawlerProcess/caching_hostname_resolver_ipv6.py
@@ -10,8 +10,7 @@ class CachingHostnameResolverSpider(scrapy.Spider):
name = "caching_hostname_resolver_spider"
async def start(self):
- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
+ yield scrapy.Request("http://[::1]")
if __name__ == "__main__":
diff --git a/tests/CrawlerProcess/default_name_resolver.py b/tests/CrawlerProcess/default_name_resolver.py
index 12b894030..4c8ce8bdd 100644
--- a/tests/CrawlerProcess/default_name_resolver.py
+++ b/tests/CrawlerProcess/default_name_resolver.py
@@ -11,8 +11,7 @@ class IPv6Spider(scrapy.Spider):
name = "ipv6_spider"
async def start(self):
- # w3lib older than 2.4.1 strips the brackets, making the URL invalid.
- yield scrapy.Request("http://[::1]", meta={"verbatim_url": True})
+ yield scrapy.Request("http://[::1]")
if __name__ == "__main__":
diff --git a/tests/test_downloadermiddleware_httpcache.py b/tests/test_downloadermiddleware_httpcache.py
index e6c593712..685d9d64a 100644
--- a/tests/test_downloadermiddleware_httpcache.py
+++ b/tests/test_downloadermiddleware_httpcache.py
@@ -11,7 +11,6 @@ from contextlib import contextmanager
from pathlib import Path
from typing import TYPE_CHECKING, Any
from unittest import mock
-from urllib.parse import urlparse
import pytest
from twisted.internet.defer import Deferred
@@ -1295,14 +1294,7 @@ class TestFilesystemStorageScope(ScopeTestMixin):
("url", "expected"),
[
("http://user:[email protected]:8080/", "www.example.com"),
- pytest.param(
- "http://[::1]/",
- "%3A%3A1",
- marks=pytest.mark.skipif(
- urlparse(Request("http://[::1]/").url).hostname != "::1",
- reason="w3lib strips the brackets of IPv6 hosts",
- ),
- ),
+ ("http://[::1]/", "%3A%3A1"),
("file:///tmp/t.txt", "_"),
],
)
diff --git a/tests/test_http_response_text.py b/tests/test_http_response_text.py
index 22c499339..5a38cc516 100644
--- a/tests/test_http_response_text.py
+++ b/tests/test_http_response_text.py
@@ -246,10 +246,7 @@ class TestTextResponse(TestResponseBase):
body=b"\xef\xbb\xbfWORD\xe3\xab",
)
assert r6.encoding == "utf-8"
- assert r6.text in {
- "WORD\ufffd\ufffd", # w3lib < 1.19.0
- "WORD\ufffd", # w3lib >= 1.19.0
- }
+ assert r6.text == "WORD\ufffd"
def test_bom_is_removed_from_body(self):
# Inferring encoding from body also cache decoded body as sideeffect,
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 247d38144..cb20f70c1 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -4,14 +4,12 @@ from typing import Any
from urllib.parse import urlparse
import pytest
-import w3lib.url
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import Request, Response
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.url import (
- _add_http_if_no_scheme,
_is_filesystem_path,
add_http_if_no_scheme,
guess_scheme,
@@ -126,83 +124,13 @@ def test_url_has_any_extension(url: str, expected: bool) -> None:
assert url_has_any_extension(url, deny_extensions) is expected
- ("url", "expected"),
- [
- ("www.example.com", "http://www.example.com"),
- ("example.com", "http://example.com"),
- ("www.example.com/some/page.html", "http://www.example.com/some/page.html"),
- ("www.example.com:80", "http://www.example.com:80"),
- ("www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
- ("www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
- (
- "username:[email protected]",
- "http://username:[email protected]",
- ),
- (
- "username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- ),
- ("http://www.example.com", "http://www.example.com"),
- ("http://example.com", "http://example.com"),
- (
- "http://www.example.com/some/page.html",
- "http://www.example.com/some/page.html",
- ),
- ("http://www.example.com:80", "http://www.example.com:80"),
- (
- "http://www.example.com/some/page#frag",
- "http://www.example.com/some/page#frag",
- ),
- (
- "http://www.example.com/do?a=1&b=2&c=3",
- "http://www.example.com/do?a=1&b=2&c=3",
- ),
- (
- "http://username:[email protected]",
- "http://username:[email protected]",
- ),
- (
- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- ),
- ("//www.example.com", "http://www.example.com"),
- ("//example.com", "http://example.com"),
- ("//www.example.com/some/page.html", "http://www.example.com/some/page.html"),
- ("//www.example.com:80", "http://www.example.com:80"),
- ("//www.example.com/some/page#frag", "http://www.example.com/some/page#frag"),
- ("//www.example.com/do?a=1&b=2&c=3", "http://www.example.com/do?a=1&b=2&c=3"),
- (
- "//username:[email protected]",
- "http://username:[email protected]",
- ),
- (
- "//username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- "http://username:[email protected]:80/some/page/do?a=1&b=2&c=3#frag",
- ),
- ("https://www.example.com", "https://www.example.com"),
- ("ftp://www.example.com", "ftp://www.example.com"),
- ],
-)
-def test_add_http_if_no_scheme(url: str, expected: str) -> None:
- assert _add_http_if_no_scheme(url) == expected
-
-
-def test_add_http_if_no_scheme_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(
- w3lib.url, "add_http_if_no_scheme", lambda url: url, raising=False
- )
+def test_add_http_if_no_scheme_deprecated() -> None:
with pytest.warns(
ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"
):
assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
-def test_add_http_if_no_scheme_not_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.delattr(w3lib.url, "add_http_if_no_scheme", raising=False)
- assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
-
-
@pytest.mark.parametrize(
("url", "expected"),
[
diff --git a/tests/utils/bases/download_handlers_http.py b/tests/utils/bases/download_handlers_http.py
index de609312b..b151e7bb3 100644
--- a/tests/utils/bases/download_handlers_http.py
+++ b/tests/utils/bases/download_handlers_http.py
@@ -40,10 +40,7 @@ from scrapy.http import (
Response,
TextResponse,
)
-from scrapy.utils._deps_compat import (
- TWISTED_TLS_LIMITS_OFFBY1,
- W3LIB_STRIPS_IPV6_BRACKETS,
-)
+from scrapy.utils._deps_compat import TWISTED_TLS_LIMITS_OFFBY1
from scrapy.utils.defer import deferred_from_coro, maybe_deferred_to_future
from scrapy.utils.misc import build_from_crawler
from scrapy.utils.reactor import is_reactor_installed
@@ -249,9 +246,6 @@ class TestHttpBase(ABC):
@pytest.mark.skipif(
not ipv6_loopback_available(), reason="IPv6 loopback is not available"
)
- @pytest.mark.skipif(
- W3LIB_STRIPS_IPV6_BRACKETS, reason="https://github.com/scrapy/w3lib/pull/253"
- )
@coroutine_test
async def test_download_ipv6_literal(self, mockserver_ipv6: MockServer) -> None:
"""Download from a URL whose host is an IPv6 address literal."""
@@ -2134,9 +2128,6 @@ class TestMitmProxyBase(ABC):
@pytest.mark.skipif(
not ipv6_loopback_available(), reason="IPv6 loopback is not available"
)
- @pytest.mark.skipif(
- W3LIB_STRIPS_IPV6_BRACKETS, reason="https://github.com/scrapy/w3lib/pull/253"
- )
@pytest.mark.parametrize("proxy_server", PROXY_KINDS, indirect=True)
@pytest.mark.parametrize(
"https_dest", [False, True], ids=["HTTP dest", "HTTPS dest"]