Scrapy, a fast high-level web crawling & scraping framework for Python.
Move to formerly and warnings (#7894)
Adriane83e45b7decd
diff --git a/pyproject.toml b/pyproject.tomlindex 6d009005a..5e3f603c3 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -13,6 +13,7 @@ dependencies = ["cryptography>=41.0.5","cssselect>=1.2.0","defusedxml>=0.7.1",+ "formerly>=0.1.0","itemadapter>=0.1.0","itemloaders>=1.0.1","lxml>=4.6.4",@@ -24,6 +25,7 @@ dependencies = ["queuelib>=1.6.1","service_identity>=24.2.0","tldextract",+ 'typing-extensions>=4.5.0; python_version < "3.13"',"w3lib>=2.1.1","zope.interface>=5.1.0",# Platform-specific dependencies@@ -224,10 +226,10 @@ disable = ["disallowed-name","duplicate-code", # https://github.com/pylint-dev/pylint/issues/214"fixme",- "inherit-non-class", # false positives with create_deprecated_class()+ "inherit-non-class", # false positives with formerly.deprecated_class()"invalid-name","invalid-overridden-method",- "isinstance-second-argument-not-valid-type", # false positives with create_deprecated_class()+ "isinstance-second-argument-not-valid-type", # false positives with formerly.deprecated_class()"line-too-long","logging-format-interpolation","logging-fstring-interpolation",diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex a098414e3..5732a9351 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport warningsfrom typing import TYPE_CHECKING, Any, cast+from formerly import deprecated_classfrom OpenSSL import SSLfrom twisted.internet.ssl import (AcceptableCiphers,@@ -28,7 +29,6 @@ from scrapy.utils._ssl import (_get_tls_version_limits,_set_keylog_callback,)-from scrapy.utils.deprecate import create_deprecated_classfrom scrapy.utils.misc import build_from_crawler, load_objectif TYPE_CHECKING:@@ -177,11 +177,12 @@ def _get_creator_context(creator: Any) -> SSL.Context:return cast("SSL.Context", creator._ctx)-ScrapyClientContextFactory = create_deprecated_class(+ScrapyClientContextFactory = deprecated_class("ScrapyClientContextFactory",_ScrapyClientContextFactory,- subclass_warn_message="{old} is deprecated.",- instance_warn_message="{cls} is deprecated.",+ category=ScrapyDeprecationWarning,+ subclass_message="{old} is deprecated.",+ instance_message="{cls} is deprecated.",)@@ -264,11 +265,12 @@ class _AcceptableProtocolsContextFactory:return options-AcceptableProtocolsContextFactory = create_deprecated_class(+AcceptableProtocolsContextFactory = deprecated_class("AcceptableProtocolsContextFactory",_AcceptableProtocolsContextFactory,- subclass_warn_message="{old} is deprecated.",- instance_warn_message="{cls} is deprecated.",+ category=ScrapyDeprecationWarning,+ subclass_message="{old} is deprecated.",+ instance_message="{cls} is deprecated.",)diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.pyindex 1995d735b..40d919064 100644--- a/scrapy/core/downloader/tls.py+++ b/scrapy/core/downloader/tls.py@@ -4,6 +4,7 @@ import loggingimport warningsfrom typing import TYPE_CHECKING, Any+from formerly import deprecated_classfrom OpenSSL import SSLfrom service_identity import VerificationErrorfrom service_identity.exceptions import CertificateError@@ -22,7 +23,6 @@ from twisted.internet._sslverify import ClientTLSOptionsfrom twisted.internet.ssl import AcceptableCiphers, TLSVersionfrom scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.deprecate import create_deprecated_classif TYPE_CHECKING:from collections.abc import Callable@@ -109,11 +109,12 @@ class _ScrapyClientTLSOptions(ClientTLSOptions):super()._identityVerifyingInfoCallback(connection, where, ret) # type: ignore[misc]-ScrapyClientTLSOptions = create_deprecated_class(+ScrapyClientTLSOptions = deprecated_class("ScrapyClientTLSOptions",_ScrapyClientTLSOptions,- subclass_warn_message="{old} is deprecated.",- instance_warn_message="{cls} is deprecated.",+ category=ScrapyDeprecationWarning,+ subclass_message="{old} is deprecated.",+ instance_message="{cls} is deprecated.",)diff --git a/scrapy/utils/decorators.py b/scrapy/utils/decorators.pyindex bd9c435de..f98a0ee55 100644--- a/scrapy/utils/decorators.py+++ b/scrapy/utils/decorators.py@@ -1,6 +1,7 @@from __future__ import annotationsimport inspect+import sysimport warningsfrom functools import wrapsfrom typing import TYPE_CHECKING, Any, ParamSpec, TypeVar, cast, overload@@ -12,6 +13,11 @@ from scrapy.utils.asyncio import run_in_threadfrom scrapy.utils.defer import deferred_from_corofrom scrapy.utils.python import _signature+if sys.version_info >= (3, 13):+ from warnings import deprecated as _deprecated+else:+ from typing_extensions import deprecated as _deprecated+if TYPE_CHECKING:from collections.abc import AsyncGenerator, Callable, Coroutine@@ -30,12 +36,15 @@ def deprecated() -> Callable[[Callable[_P, _T]], Callable[_P, _T]]: ...+@_deprecated(+ "scrapy.utils.decorators.deprecated() is deprecated, use warnings.deprecated()"+ " instead.",+ category=ScrapyDeprecationWarning,+)def deprecated(use_instead: Callable[_P, _T] | str | None = None,) -> Callable[_P, _T] | Callable[[Callable[_P, _T]], Callable[_P, _T]]:- """This is a decorator which can be used to mark functions- as deprecated. It will result in a warning being emitted- when the function is used."""+ """Mark a function as deprecated, so that calling it warns."""def deco(func: Callable[_P, _T]) -> Callable[_P, _T]:@wraps(func)diff --git a/scrapy/utils/deprecate.py b/scrapy/utils/deprecate.pyindex 2aaa34365..9d6cde75f 100644--- a/scrapy/utils/deprecate.py+++ b/scrapy/utils/deprecate.py@@ -3,16 +3,27 @@from __future__ import annotationsimport inspect+import sysimport warnings-from typing import TYPE_CHECKING, Any, cast, overload+from typing import TYPE_CHECKING, Any, overload++from formerly import deprecated_classfrom scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.python import get_func_args_dict+if sys.version_info >= (3, 13):+ from warnings import deprecated as _deprecated+else:+ from typing_extensions import deprecated as _deprecated+if TYPE_CHECKING:from collections.abc import Callable+_WRAPPER_MODULES = frozenset({__name__, _deprecated.__module__})++def attribute(obj: object, oldattr: str, newattr: str, version: str = "0.12") -> None:cname = obj.__class__.__name__warnings.warn(@@ -23,6 +34,11 @@ def attribute(obj: object, oldattr: str, newattr: str, version: str = "0.12") ->)+@_deprecated(+ "scrapy.utils.deprecate.create_deprecated_class() is deprecated, use"+ " formerly.deprecated_class() instead.",+ category=ScrapyDeprecationWarning,+)def create_deprecated_class(name: str,new_class: type,@@ -34,114 +50,27 @@ def create_deprecated_class(subclass_warn_message: str = "{cls} inherits from deprecated class {old}, please inherit from {new}.",instance_warn_message: str = "{cls} is deprecated, instantiate {new} instead.",) -> type:- """- Return a "deprecated" class that causes its subclasses to issue a warning.- Subclasses of ``new_class`` are considered subclasses of this class.- It also warns when the deprecated class is instantiated, but do not when- its subclasses are instantiated.-- It can be used to rename a base class in a library. For example, if we- have-- .. code-block:: python-- class OldName(SomeClass): ...-- and we want to rename it to NewName, we can do the following:-- .. code-block:: python-- class NewName(SomeClass): ...--- OldName = create_deprecated_class("OldName", NewName)-- Then, if user class inherits from OldName, warning is issued. Also, if- some code uses ``issubclass(sub, OldName)`` or ``isinstance(sub(), OldName)``- checks they'll still return True if sub is a subclass of NewName instead of- OldName.- """-- # https://github.com/python/mypy/issues/4177- class DeprecatedClass(new_class.__class__): # type: ignore[misc,name-defined]- # pylint: disable=no-self-argument- deprecated_class: type | None = None- warned_on_subclass: bool = False-- def __new__( # pylint: disable=bad-classmethod-argument- metacls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]- ) -> type:- cls: type = super().__new__(metacls, name, bases, clsdict_)- if metacls.deprecated_class is None:- metacls.deprecated_class = cls- return cls-- def __init__(cls, name: str, bases: tuple[type, ...], clsdict_: dict[str, Any]):- meta = cls.__class__- old = meta.deprecated_class- if old in bases and not (warn_once and meta.warned_on_subclass):- meta.warned_on_subclass = True- msg = subclass_warn_message.format(- cls=_clspath(cls),- old=_clspath(old, old_class_path),- new=_clspath(new_class, new_class_path),- )- if warn_once:- msg += " (warning only on first subclass, there may be others)"- warnings.warn(msg, warn_category, stacklevel=2)- super().__init__(name, bases, clsdict_)-- # see https://www.python.org/dev/peps/pep-3119/#overloading-isinstance-and-issubclass- # and https://docs.python.org/reference/datamodel.html#customizing-instance-and-subclass-checks- # for implementation details- def __instancecheck__(cls, inst: Any) -> bool:- return any(cls.__subclasscheck__(c) for c in (type(inst), inst.__class__))-- def __subclasscheck__(cls, sub: type) -> bool:- if cls is not DeprecatedClass.deprecated_class:- # we should do the magic only if second `issubclass` argument- # is the deprecated class itself - subclasses of the- # deprecated class should not use custom `__subclasscheck__`- # method.- return cast("bool", super().__subclasscheck__(sub))-- if not inspect.isclass(sub):- raise TypeError("issubclass() arg 1 must be a class")-- mro = getattr(sub, "__mro__", ())- return any(c in {cls, new_class} for c in mro)-- def __call__(cls, *args: Any, **kwargs: Any) -> Any:- old = DeprecatedClass.deprecated_class- if cls is old:- msg = instance_warn_message.format(- cls=_clspath(cls, old_class_path),- new=_clspath(new_class, new_class_path),- )- warnings.warn(msg, warn_category, stacklevel=2)- return super().__call__(*args, **kwargs)-- deprecated_cls = DeprecatedClass(name, (new_class,), clsdict or {})-- try:- frm = inspect.stack()[1]- parent_module = inspect.getmodule(frm[0])- if parent_module is not None:- deprecated_cls.__module__ = parent_module.__name__- except Exception as e:- # Sometimes inspect.stack() fails (e.g. when the first import of- # deprecated class is in jinja2 template). __module__ attribute is not- # important enough to raise an exception as users may be unable- # to fix inspect.stack() errors.- warnings.warn(f"Error detecting parent module: {e!r}", stacklevel=2)-- return deprecated_cls---def _clspath(cls: type, forced: str | None = None) -> str:- if forced is not None:- return forced- return f"{cls.__module__}.{cls.__name__}"+ """Return a deprecated alias of *new_class* named *name*."""+ cls: type = deprecated_class(+ name,+ new_class,+ namespace=clsdict,+ category=warn_category,+ warn_once=warn_once,+ old_path=old_class_path,+ new_path=new_class_path,+ subclass_message=subclass_warn_message,+ instance_message=instance_warn_message,+ )+ # deprecated_class() takes the module of the alias from its calling frame,+ # which is this function and the decorator wrapping it, so skip past both.+ frame = inspect.currentframe()+ assert frame is not None+ while frame.f_globals.get("__name__") in _WRAPPER_MODULES:+ assert frame.f_back is not None+ frame = frame.f_back+ cls.__module__ = frame.f_globals.get("__name__", cls.__module__)+ return clsDEPRECATION_RULES: list[tuple[str, str]] = []diff --git a/tests/test_utils_decorators.py b/tests/test_utils_decorators.pyindex 92677882e..d4e9e8129 100644--- a/tests/test_utils_decorators.py+++ b/tests/test_utils_decorators.py@@ -18,7 +18,12 @@ if TYPE_CHECKING:class TestDeprecated:def test_warns_and_still_calls(self):- @deprecated()+ with pytest.warns(+ ScrapyDeprecationWarning, match=r"decorators\.deprecated\(\) is deprecated"+ ):+ decorate = deprecated()++ @decoratedef add(a: int, b: int) -> int:return a + b@@ -30,7 +35,11 @@ class TestDeprecated:assert result == 5def test_use_instead_in_message(self):- @deprecated(use_instead="other_function")+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ decorate = deprecated(use_instead="other_function")++ @decoratedef old() -> None:return None@@ -41,10 +50,13 @@ class TestDeprecated:old()def test_applied_without_parentheses(self):- @deprecateddef square(x: int) -> int:return x * x+ with warnings.catch_warnings():+ warnings.simplefilter("ignore", ScrapyDeprecationWarning)+ square = deprecated(square)+with pytest.warns(ScrapyDeprecationWarning, match=r"Call to deprecated function square\.") as record:diff --git a/tests/test_utils_deprecate.py b/tests/test_utils_deprecate.pyindex 5aa221414..2c1913669 100644--- a/tests/test_utils_deprecate.py+++ b/tests/test_utils_deprecate.py@@ -1,6 +1,5 @@from __future__ import annotations-import inspectimport warningsfrom unittest import mock@@ -10,252 +9,47 @@ from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.utils.deprecate import attribute, create_deprecated_class, update_classpath-class MyWarning(UserWarning):+class NewName:pass-class SomeBaseClass:- pass---class NewName(SomeBaseClass):- pass---class TestWarnWhenSubclassed:- def test_no_warning_on_definition(self):- with warnings.catch_warnings():- warnings.simplefilter("error", category=ScrapyDeprecationWarning)- create_deprecated_class("Deprecated", NewName)-- def test_subclassing_warning_message(self):- msg = (- r"tests\.test_utils_deprecate\.UserClass inherits from "- r"deprecated class tests\.test_utils_deprecate\.Deprecated, "- r"please inherit from tests\.test_utils_deprecate\.NewName."- r" \(warning only on first subclass, there may be others\)"- )- Deprecated = create_deprecated_class(- "Deprecated", NewName, warn_category=MyWarning- )- with pytest.warns(MyWarning, match=msg) as w:-- class UserClass(Deprecated): # type: ignore[misc, valid-type]- pass-- assert w[0].lineno == inspect.getsourcelines(UserClass)[1]-- def test_custom_class_paths(self):- Deprecated = create_deprecated_class(- "Deprecated",- NewName,- new_class_path="foo.NewClass",- old_class_path="bar.OldClass",- warn_category=MyWarning,- )-- with pytest.warns(- MyWarning,- match=r"UserClass inherits from deprecated class bar\.OldClass, please inherit from foo\.NewClass",- ):-- class UserClass(Deprecated): # type: ignore[misc, valid-type]- pass-- with pytest.warns(- MyWarning,- match=r"bar\.OldClass is deprecated, instantiate foo\.NewClass instead",- ):- _ = Deprecated()-- def test_subclassing_warns_only_on_direct_children(self):- Deprecated = create_deprecated_class(- "Deprecated", NewName, warn_once=False, warn_category=MyWarning- )-- with pytest.warns(- MyWarning,- match="UserClass inherits from deprecated class",- ):-- class UserClass(Deprecated): # type: ignore[misc, valid-type]- pass-- with warnings.catch_warnings():- warnings.simplefilter("error", MyWarning)-- class NoWarnOnMe(UserClass):- pass-- def test_subclassing_warns_once_by_default(self):- Deprecated = create_deprecated_class(- "Deprecated", NewName, warn_category=MyWarning- )-- with pytest.warns(- MyWarning,- match="UserClass inherits from deprecated class",- ):-- class UserClass(Deprecated): # type: ignore[misc, valid-type]- pass-- with warnings.catch_warnings():- warnings.simplefilter("error", MyWarning)-- class FooClass(Deprecated): # type: ignore[misc, valid-type]- pass-- class BarClass(Deprecated): # type: ignore[misc, valid-type]- pass-- def test_warning_on_instance(self):- Deprecated = create_deprecated_class(- "Deprecated", NewName, warn_category=MyWarning- )-- with pytest.warns(- MyWarning,- match=r"tests\.test_utils_deprecate\.Deprecated is deprecated, "- r"instantiate tests\.test_utils_deprecate\.NewName instead\.",- ) as w:- _, lineno = Deprecated(), inspect.getlineno(inspect.currentframe()) # type: ignore[arg-type]- assert len(w) == 1- assert w[0].lineno == lineno-- # ignore subclassing warnings- with warnings.catch_warnings():- warnings.simplefilter("ignore", MyWarning)+def test_attribute():+ with pytest.warns(+ ScrapyDeprecationWarning,+ match=r"NewName\.old attribute is deprecated and will be no longer supported"+ r" in Scrapy 1\.0, use NewName\.new attribute instead",+ ):+ attribute(NewName(), "old", "new", version="1.0")- class UserClass(Deprecated): # type: ignore[misc, valid-type]- pass- with warnings.catch_warnings():- warnings.simplefilter("error", MyWarning)- UserClass() # subclass instances don't warn-- def test_warning_auto_message(self):- Deprecated = create_deprecated_class("Deprecated", NewName)+class TestCreateDeprecatedClass:+ def test_warns_about_itself(self):with pytest.warns(- ScrapyDeprecationWarning,- match=r"UserClass2 inherits from deprecated class tests\.test_utils_deprecate\.Deprecated, please inherit from tests\.test_utils_deprecate\.NewName",+ ScrapyDeprecationWarning, match=r"create_deprecated_class\(\) is deprecated"):+ create_deprecated_class("Deprecated", NewName)- class UserClass2(Deprecated): # type: ignore[misc, valid-type]- pass-- def test_issubclass(self):- with warnings.catch_warnings():- warnings.simplefilter("ignore", ScrapyDeprecationWarning)- DeprecatedName = create_deprecated_class("DeprecatedName", NewName)-- class UpdatedUserClass1(NewName):- pass-- class UpdatedUserClass1a(NewName):- pass-- class OutdatedUserClass1(DeprecatedName): # type: ignore[misc, valid-type]- pass-- class OutdatedUserClass1a(DeprecatedName): # type: ignore[misc, valid-type]- pass-- class UnrelatedClass:- pass-- assert issubclass(UpdatedUserClass1, NewName)- assert issubclass(UpdatedUserClass1a, NewName)- assert issubclass(UpdatedUserClass1, DeprecatedName)- assert issubclass(UpdatedUserClass1a, DeprecatedName)- assert issubclass(OutdatedUserClass1, DeprecatedName)- assert not issubclass(UnrelatedClass, DeprecatedName)- assert not issubclass(OutdatedUserClass1, OutdatedUserClass1a)- assert not issubclass(OutdatedUserClass1a, OutdatedUserClass1)-- with pytest.raises(TypeError):- issubclass(object(), DeprecatedName) # type: ignore[arg-type]-- def test_isinstance(self):- with warnings.catch_warnings():- warnings.simplefilter("ignore", ScrapyDeprecationWarning)- DeprecatedName = create_deprecated_class("DeprecatedName", NewName)-- class UpdatedUserClass2(NewName):- pass-- class UpdatedUserClass2a(NewName):- pass-- class OutdatedUserClass2(DeprecatedName): # type: ignore[misc, valid-type]- pass-- class OutdatedUserClass2a(DeprecatedName): # type: ignore[misc, valid-type]- pass-- class UnrelatedClass:- pass-- assert isinstance(UpdatedUserClass2(), NewName)- assert isinstance(UpdatedUserClass2a(), NewName)- assert isinstance(UpdatedUserClass2(), DeprecatedName)- assert isinstance(UpdatedUserClass2a(), DeprecatedName)- assert isinstance(OutdatedUserClass2(), DeprecatedName)- assert isinstance(OutdatedUserClass2a(), DeprecatedName)- assert not isinstance(OutdatedUserClass2a(), OutdatedUserClass2)- assert not isinstance(OutdatedUserClass2(), OutdatedUserClass2a)- assert not isinstance(UnrelatedClass(), DeprecatedName)-- def test_clsdict(self):+ def test_returns_a_working_alias(self):with warnings.catch_warnings():warnings.simplefilter("ignore", ScrapyDeprecationWarning)- Deprecated = create_deprecated_class("Deprecated", NewName, {"foo": "bar"})-- assert Deprecated.foo == "bar" # type: ignore[attr-defined]-- def test_deprecate_a_class_with_custom_metaclass(self):- Meta1 = type("Meta1", (type,), {})- New = Meta1("New", (), {})- create_deprecated_class("Deprecated", New)-- def test_deprecate_subclass_of_deprecated_class(self):- with warnings.catch_warnings():- warnings.simplefilter("error", MyWarning)Deprecated = create_deprecated_class(- "Deprecated", NewName, warn_category=MyWarning- )- AlsoDeprecated = create_deprecated_class(- "AlsoDeprecated",- Deprecated,- new_class_path="foo.Bar",- warn_category=MyWarning,+ "Deprecated", NewName, {"foo": "bar"}, warn_once=False)- with pytest.warns(- MyWarning,- match=r"AlsoDeprecated is deprecated, instantiate foo\.Bar instead",- ):- AlsoDeprecated()+ assert Deprecated.__module__ == __name__+ assert Deprecated.foo == "bar" # type: ignore[attr-defined]with pytest.warns(- MyWarning,- match=r"UserClass inherits from deprecated class tests\.test_utils_deprecate\.AlsoDeprecated, please inherit from foo\.Bar",+ ScrapyDeprecationWarning,+ match=r"tests\.test_utils_deprecate\.UserClass inherits from deprecated"+ r" class tests\.test_utils_deprecate\.Deprecated, please inherit from"+ r" tests\.test_utils_deprecate\.NewName\.",):- class UserClass(AlsoDeprecated): # type: ignore[misc, valid-type]+ class UserClass(Deprecated): # type: ignore[misc, valid-type]pass- def test_inspect_stack(self):- with (- mock.patch("inspect.stack", side_effect=IndexError),- pytest.warns(UserWarning, match="Error detecting parent module"),- ):- create_deprecated_class("DeprecatedName", NewName)-- def test_unknown_parent_module(self):- with mock.patch("inspect.getmodule", return_value=None):- cls = create_deprecated_class("DeprecatedName", NewName)- assert cls.__module__ == "scrapy.utils.deprecate"+ assert issubclass(UserClass, Deprecated)@mock.patch(diff --git a/tox.ini b/tox.iniindex 732225b5f..5931e2ffc 100644--- a/tox.ini+++ b/tox.ini@@ -148,6 +148,7 @@ deps =charset-normalizer==3.4.0cryptography==41.0.5cssselect==1.2.0+ formerly==0.1.0itemadapter==0.1.0lxml==4.6.4parsel==1.8.1@@ -155,6 +156,7 @@ deps =pyOpenSSL==24.3.0queuelib==1.6.1service_identity==24.2.0+ typing-extensions==4.5.0w3lib==2.1.1zope.interface==5.1.0{[test-requirements]deps}