Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Drop w3lib.url reexports.

Andrey Rakhmatullin5fc40f07f3c3
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.py
index 1c05ac20e..8f75e2618 100644
--- a/scrapy/utils/url.py
+++ b/scrapy/utils/url.py
@@ -6,34 +6,10 @@ library.
from __future__ import annotations
import re
-import warnings
-from importlib import import_module
-from typing import TYPE_CHECKING, Any, TypeAlias
+from typing import TYPE_CHECKING, TypeAlias
from urllib.parse import ParseResult, urlparse, urlunparse
-from w3lib.url import __all__ as _public_w3lib_objects
-from w3lib.url import any_to_uri as _any_to_uri
-from w3lib.url import parse_url as _parse_url
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-_DEPRECATED_NAMES: frozenset[str] = frozenset(
- {"_unquotepath", "_safe_chars", "parse_url", *_public_w3lib_objects}
-)
-
-
-def __getattr__(name: str) -> Any:
- if name in _DEPRECATED_NAMES:
- obj_type = "attribute" if name == "_safe_chars" else "function"
- warnings.warn(
- f"The scrapy.utils.url.{name} {obj_type} is deprecated, use w3lib.url.{name} instead.",
- ScrapyDeprecationWarning,
- stacklevel=2,
- )
- return getattr(import_module("w3lib.url"), name)
-
- raise AttributeError
-
+from w3lib.url import any_to_uri, parse_url
if TYPE_CHECKING:
from collections.abc import Iterable
@@ -45,7 +21,7 @@ UrlT: TypeAlias = str | bytes | ParseResult
def url_is_from_any_domain(url: UrlT, domains: Iterable[str]) -> bool:
"""Return True if the url belongs to any of the given domains"""
- host = _parse_url(url).netloc.lower()
+ host = parse_url(url).netloc.lower()
if not host:
return False
return any((host == d) or (host.endswith(f".{d}")) for d in map(str.lower, domains))
@@ -64,7 +40,7 @@ def url_is_from_spider(url: UrlT, spider: type[Spider]) -> bool:
def url_has_any_extension(url: UrlT, extensions: Iterable[str]) -> bool:
"""Return True if the url ends with one of the extensions provided"""
- lowercase_path = _parse_url(url).path.lower()
+ lowercase_path = parse_url(url).path.lower()
return any(lowercase_path.endswith(ext) for ext in extensions)
@@ -125,7 +101,7 @@ def guess_scheme(url: str) -> str:
"""Add an URL scheme if missing: file:// for filepath-like input or
http:// otherwise."""
if _is_filesystem_path(url):
- return _any_to_uri(url)
+ return any_to_uri(url)
return add_http_if_no_scheme(url)
diff --git a/tests/test_utils_url.py b/tests/test_utils_url.py
index bcbbe74a3..19a31c353 100644
--- a/tests/test_utils_url.py
+++ b/tests/test_utils_url.py
@@ -1,13 +1,9 @@
-import warnings
-from importlib import import_module
-
import pytest
from scrapy.linkextractors import IGNORED_EXTENSIONS
from scrapy.spiders import Spider
-from scrapy.utils.url import ( # type: ignore[attr-defined]
+from scrapy.utils.url import (
_is_filesystem_path,
- _public_w3lib_objects,
add_http_if_no_scheme,
guess_scheme,
strip_url,
@@ -446,23 +442,3 @@ class TestStripUrl:
)
def test__is_filesystem_path(path: str, expected: bool) -> None:
assert _is_filesystem_path(path) == expected
-
-
- "obj_name",
- [
- "_unquotepath",
- "_safe_chars",
- "parse_url",
- *_public_w3lib_objects,
- ],
-)
-def test_deprecated_imports_from_w3lib(obj_name: str) -> None:
- with warnings.catch_warnings(record=True) as warns:
- obj_type = "attribute" if obj_name == "_safe_chars" else "function"
- message = f"The scrapy.utils.url.{obj_name} {obj_type} is deprecated, use w3lib.url.{obj_name} instead."
-
- getattr(import_module("scrapy.utils.url"), obj_name)
-
- assert isinstance(warns[0].message, Warning)
- assert message in warns[0].message.args