Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate scrapy.utils.url.add_http_if_no_scheme() in favor of w3lib (#8189)

Adriancbc2de9699cd
diff --git a/scrapy/core/downloader/handlers/_base_streaming.py b/scrapy/core/downloader/handlers/_base_streaming.py
index 6e4dd6870..c7f4c82d8 100644
--- a/scrapy/core/downloader/handlers/_base_streaming.py
+++ b/scrapy/core/downloader/handlers/_base_streaming.py
@@ -23,7 +23,7 @@ from scrapy.utils._download_handlers import (
normalize_bind_address,
)
from scrapy.utils.asyncio import is_asyncio_available
-from scrapy.utils.url import add_http_if_no_scheme
+from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
@@ -286,7 +286,7 @@ class BaseStreamingDownloadHandler(BaseHttpDownloadHandler, ABC, Generic[_Respon
proxy: str | None = request.meta.get("proxy")
if not proxy:
return None, None
- proxy = add_http_if_no_scheme(proxy)
+ proxy = _add_http_if_no_scheme(proxy)
auth_header: bytes | None = request.headers.get(b"Proxy-Authorization")
return proxy, auth_header.decode("ascii") if auth_header else None
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index f3eaa3411..37b27c8a6 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -62,7 +62,7 @@ from scrapy.utils.defer import maybe_deferred_to_future
from scrapy.utils.deprecate import warn_on_deprecated_spider_attribute
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_bytes, to_unicode
-from scrapy.utils.url import add_http_if_no_scheme
+from scrapy.utils.url import _add_http_if_no_scheme
from ._base_http import BaseHttpDownloadHandler
@@ -456,7 +456,7 @@ class _ScrapyAgent:
bindaddress = normalize_bind_address(bindaddress)
proxy = request.meta.get("proxy")
if proxy:
- proxy = add_http_if_no_scheme(proxy)
+ proxy = _add_http_if_no_scheme(proxy)
proxy_parsed = urlparse(proxy)
proxy_host = proxy_parsed.hostname
proxy_port = proxy_parsed.port
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 21ef4aac7..de2e38728 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -10,8 +10,11 @@ import warnings
from typing import TYPE_CHECKING, TypeAlias
from urllib.parse import ParseResult, urlparse, urlunparse
+import w3lib.url
from w3lib.url import any_to_uri, parse_url
+from scrapy.exceptions import ScrapyDeprecationWarning
+
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -57,17 +60,25 @@ def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
return any(lowercase_path.endswith(ext) for ext in extensions)
-def add_http_if_no_scheme(url: str) -> str:
- """Add http as the default scheme if it is missing from the url."""
- match = re.match(r"^\w+://", url, flags=re.IGNORECASE)
- if not match:
- parts = urlparse(url)
- scheme = "http:" if parts.netloc else "http://"
+def _add_http_if_no_scheme(url: str) -> str:
+ if not re.match(r"^\w+://", url, flags=re.IGNORECASE):
+ scheme = "http:" if urlparse(url).netloc else "http://"
url = scheme + url
-
return url
+def add_http_if_no_scheme(url: str) -> str:
+ """Add http as the default scheme if it is missing from the url."""
+ if hasattr(w3lib.url, "add_http_if_no_scheme"):
+ warnings.warn(
+ "scrapy.utils.url.add_http_if_no_scheme() is deprecated, use "
+ "w3lib.url.add_http_if_no_scheme() instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return _add_http_if_no_scheme(url)
+
+
def _is_posix_path(string: str) -> bool:
return bool(
re.match(
@@ -115,7 +126,7 @@ def guess_scheme(url: str) -> str:
http:// otherwise."""
if _is_filesystem_path(url):
return any_to_uri(url)
- return add_http_if_no_scheme(url)
+ return _add_http_if_no_scheme(url)
def strip_url(
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index 403fd6176..830d691e2 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,10 +1,13 @@
from __future__ import annotations
import pytest
+import w3lib.url
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
from scrapy.utils.url import (
+ _add_http_if_no_scheme,
_is_filesystem_path,
add_http_if_no_scheme,
guess_scheme,
@@ -162,7 +165,22 @@ def test_url_has_any_extension(url: str, expected: bool) -> None:
],
)
def test_add_http_if_no_scheme(url: str, expected: str) -> None:
- assert add_http_if_no_scheme(url) == expected
+ assert _add_http_if_no_scheme(url) == expected
+
+
+def test_add_http_if_no_scheme_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(
+ w3lib.url, "add_http_if_no_scheme", lambda url: url, raising=False
+ )
+ with pytest.warns(
+ ScrapyDeprecationWarning, match="use w3lib.url.add_http_if_no_scheme"
+ ):
+ assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
+
+
+def test_add_http_if_no_scheme_not_deprecated(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.delattr(w3lib.url, "add_http_if_no_scheme", raising=False)
+ assert add_http_if_no_scheme("www.example.com") == "http://www.example.com"
@pytest.mark.parametrize(