Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate scrapy.utils.url.add_http_if_no_scheme() in favor of w3lib (#8189)
Adriancbc2de9699cd
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.pyindex 6e4dd6870..c7f4c82d8 100644--- a/scrapy/core/downloader/handlers/_base_streaming.py+++ b/scrapy/core/downloader/handlers/_base_streaming.py@@ -23,7 +23,7 @@ from scrapy.utils._download_handlers import (normalize_bind_address,)from scrapy.utils.asyncio import is_asyncio_available-from scrapy.utils.url import add_http_if_no_scheme+from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandler@@ -286,7 +286,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Responproxy: str | None = request.meta.get("proxy")if not proxy:return None, None- proxy = add_http_if_no_scheme(proxy)+ proxy = _add_http_if_no_scheme(proxy)auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")return proxy, auth_header.decode("ascii") if auth_header else Nonediff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex f3eaa3411..37b27c8a6 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -62,7 +62,7 @@ from scrapy.utils.defer import maybe_deferred_to_futurefrom scrapy.utils.deprecate import warn_on_deprecated_spider_attributefrom scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.python import to_bytes, to_unicode-from scrapy.utils.url import add_http_if_no_scheme+from scrapy.utils.url import _add_http_if_no_schemefrom ._base_http import BaseHttpDownloadHandler@@ -456,7 +456,7 @@ class _ScrapyAgent:bindaddress = normalize_bind_address(bindaddress)proxy = request.meta.get("proxy")if proxy:- proxy = add_http_if_no_scheme(proxy)+ proxy = _add_http_if_no_scheme(proxy)proxy_parsed = urlparse(proxy)proxy_host = proxy_parsed.hostnameproxy_port = proxy_parsed.portdiff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 21ef4aac7..de2e38728 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -10,8 +10,11 @@ import warningsfrom typing import TYPE_CHECKING, TypeAliasfrom urllib.parse import ParseResult, urlparse, urlunparse+import w3lib.urlfrom w3lib.url import any_to_uri, parse_url+from scrapy.exceptions import ScrapyDeprecationWarning+if TYPE_CHECKING:from collections.abc import Iterable@@ -57,17 +60,25 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:return any(lowercase_path.endswith(ext) for ext in extensions)-def add_http_if_no_scheme(url: str) -> str:- """Add http as the default scheme if it is missing from the url."""- match = re.match(r"^\w+://", url, flags=re.IGNORECASE)- if not match:- parts = urlparse(url)- scheme = "http:" if parts.netloc else "http://"+def _add_http_if_no_scheme(url: str) -> str:+ if not re.match(r"^\w+://", url, flags=re.IGNORECASE):+ scheme = "http:" if urlparse(url).netloc else "http://"url = scheme + url-return url+def add_http_if_no_scheme(url: str) -> str:+ """Add http as the default scheme if it is missing from the url."""+ if hasattr(w3lib.url, "add_http_if_no_scheme"):+ warnings.warn(+ "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "+ "w3lib.url.add_http_if_no_scheme() instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _add_http_if_no_scheme(url)++def _is_posix_path(string: str) -> bool:return bool(re.match(@@ -115,7 +126,7 @@ def guess_scheme(url: str) -> str:http:// otherwise."""if _is_filesystem_path(url):return any_to_uri(url)- return add_http_if_no_scheme(url)+ return _add_http_if_no_scheme(url)def strip_url(diff --git a/tests/test_utils_url.py b/tests/test_utils_url.pyindex 403fd6176..830d691e2 100644--- a/tests/test_utils_url.py+++ b/tests/test_utils_url.py@@ -1,10 +1,13 @@from __future__ import annotationsimport pytest+import w3lib.url+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.linkextractors import IGNORED_EXTENSIONSfrom scrapy.spiders import Spiderfrom scrapy.utils.url import (+ _add_http_if_no_scheme,_is_filesystem_path,add_http_if_no_scheme,guess_scheme,@@ -162,7 +165,22 @@ def test_url_has_any_extension(url: str, expected: bool) -> None:],)def test_add_http_if_no_scheme(url: str, expected: str) -> None:- assert add_http_if_no_scheme(url) == expected+ assert _add_http_if_no_scheme(url) == expected+++def test_add_http_if_no_scheme_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(+ w3lib.url, "add_http_if_no_scheme", lambda url: url, raising=False+ )+ with pytest.warns(+ ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"+ ):+ assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"+++def test_add_http_if_no_scheme_not_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.delattr(w3lib.url, "add_http_if_no_scheme", raising=False)+ assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"@pytest.mark.parametrize(