Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove non-Twisted uses of zope.interface (#7731)
Andrey Rakhmatullin394c2797f340
diff --git a/docs/news.rst b/docs/news.rstindex 63f0d69e4..4ebab3c6a 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,28 @@Release notes=============+Scrapy VERSION (unreleased)+---------------------------++Backward-incompatible changes+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++- The following runtime usage of zope.interface_ interfaces is removed:++ - :class:`~scrapy.spiderloader.SpiderLoader` and+ :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked+ as implementing the ``ISpiderLoader`` interface.++ - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the+ configured spider loader implements the ``ISpiderLoader`` interface.++ - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,+ :class:`~scrapy.extensions.feedexport.FileFeedStorage` and+ :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer+ marked as implementing the ``IFeedStorage`` interface.++ (:issue:`6585`, :issue:`7731`)+.. _release-2.17.0:Scrapy 2.17.0 (2026-07-07)@@ -2643,7 +2665,7 @@ Deprecation removals(:issue:`6109`, :issue:`6116`)- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that- does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface+ does not implement the ``ISpiderLoader`` interfacewill now raise a :exc:`zope.interface.verify.DoesNotImplement` exception atrun time. Non-compliant classes have been triggering a deprecation warningsince Scrapy 1.0.0.diff --git a/docs/topics/api.rst b/docs/topics/api.rstindex 19082d9d7..7ff8d3464 100644--- a/docs/topics/api.rst+++ b/docs/topics/api.rst@@ -172,46 +172,15 @@ SpiderLoader API.. module:: scrapy.spiderloader:synopsis: The spider loader-.. class:: SpiderLoader+Custom spider loaders can be employed by specifying their path in the+:setting:`SPIDER_LOADER_CLASS` project setting. They must implement+:class:`SpiderLoaderProtocol`.- This class is in charge of retrieving and handling the spider classes- defined across the project.-- Custom spider loaders can be employed by specifying their path in the- :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement- the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an- errorless execution.-- .. method:: from_settings(settings)-- This class method is used by Scrapy to create an instance of the class.- It's called with the current project settings, and it loads the spiders- found recursively in the modules of the :setting:`SPIDER_MODULES`- setting.-- :param settings: project settings- :type settings: :class:`~scrapy.settings.Settings` instance-- .. method:: load(spider_name)-- Get the Spider class with the given name. It'll look into the previously- loaded spiders for a spider class with name ``spider_name`` and will raise- a KeyError if not found.-- :param spider_name: spider class name- :type spider_name: str-- .. method:: list()-- Get the names of the available spiders in the project.-- .. method:: find_by_request(request)-- List the spiders' names that can handle the given request. Will try to- match the request's url against the domains of the spiders.+.. autoclass:: SpiderLoaderProtocol+ :members:- :param request: queried request- :type request: :class:`~scrapy.Request` instance+.. autoclass:: SpiderLoader+ :members:.. autoclass:: DummySpiderLoaderdiff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.pyindex a934cbbc7..ef948997d 100644--- a/scrapy/core/downloader/contextfactory.py+++ b/scrapy/core/downloader/contextfactory.py@@ -13,7 +13,6 @@ from twisted.internet.ssl import (from twisted.web.client import BrowserLikePolicyForHTTPSfrom twisted.web.iweb import IPolicyForHTTPSfrom zope.interface.declarations import implementer-from zope.interface.verify import verifyObjectfrom scrapy.core.downloader.tls import (_TWISTED_VERSION_MAP,@@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory:# all of this with _ScrapyClientContextFactory.acceptableProtocols.def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):- verifyObject(IPolicyForHTTPS, context_factory)self._wrapped_context_factory: Any = context_factoryself._acceptable_protocols: list[bytes] = acceptable_protocolsdiff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex 0d9b016a3..678a29e2e 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparsefrom twisted.internet.defer import Deferred, DeferredListfrom w3lib.url import file_uri_to_path-from zope.interface import Interface, implementer+from zope.interface import Interfacefrom scrapy import Spider, signalsfrom scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning@@ -115,25 +115,18 @@ class ItemFilter:return True # accept all items by default-class IFeedStorage(Interface): # type: ignore[misc]- """Interface that all Feed Storages must implement"""-+class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover# pylint: disable=no-self-argument- def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called- """Initialize the storage with the parameters given in the URI and the- feed-specific options (see :setting:`FEEDS`)"""+ def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called- def open(spider): # type: ignore[no-untyped-def]- """Open the storage for the given spider. It must return a file-like- object that will be used for the exporters"""+ def open(spider): ... # type: ignore[no-untyped-def]- def store(file): # type: ignore[no-untyped-def]- """Store the given file stream"""+ def store(file): ... # type: ignore[no-untyped-def]class FeedStorageProtocol(Protocol):- """Reimplementation of ``IFeedStorage`` that can be used in type hints."""+ """Protocol that all Feed Storages must follow."""def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):"""Initialize the storage with the parameters given in the URI and the@@ -147,7 +140,6 @@ class FeedStorageProtocol(Protocol):"""Store the given file stream"""-@implementer(IFeedStorage)class BlockingFeedStorage(ABC):def open(self, spider: Spider) -> IO[bytes]:path = spider.crawler.settings["FEED_TEMPDIR"]@@ -164,7 +156,6 @@ class BlockingFeedStorage(ABC):raise NotImplementedError-@implementer(IFeedStorage)class StdoutFeedStorage:def __init__(self,@@ -191,7 +182,6 @@ class StdoutFeedStorage:pass-@implementer(IFeedStorage)class FileFeedStorage:def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri@@ -758,3 +748,14 @@ class FeedExporter:feed_options.get("item_filter", ItemFilter))return item_filter_class(feed_options)+++def __getattr__(name: str) -> Any: # pragma: no cover+ if name == "IFeedStorage":+ warnings.warn(+ "scrapy.extensions.feedexport.IFeedStorage is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+ )+ return _IFeedStorage+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")diff --git a/scrapy/interfaces.py b/scrapy/interfaces.pyindex b4f1d9394..34d7bdd82 100644--- a/scrapy/interfaces.py+++ b/scrapy/interfaces.py@@ -1,19 +1,23 @@+# pragma: no file cover# pylint: disable=no-method-argument,no-self-argument+import warningsfrom zope.interface import Interface+from scrapy.exceptions import ScrapyDeprecationWarning++warnings.warn(+ "The scrapy.interfaces module is deprecated.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)+class ISpiderLoader(Interface):- def from_settings(settings):- """Return an instance of the class for the given settings"""+ def from_settings(settings): ...- def load(spider_name):- """Return the Spider class for the given spider name. If the spider- name is not found, it must raise a KeyError."""+ def load(spider_name): ...- def list():- """Return a list with the names of all spiders available in the- project"""+ def list(): ...- def find_by_request(request):- """Return the list of spiders names that can handle the given request"""+ def find_by_request(request): ...diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.pyindex 8c980fd46..244b7338f 100644--- a/scrapy/spiderloader.py+++ b/scrapy/spiderloader.py@@ -5,10 +5,8 @@ import warningsfrom collections import defaultdictfrom typing import TYPE_CHECKING, Protocol, cast-from zope.interface import implementer-from zope.interface.verify import verifyClass--from scrapy.interfaces import ISpiderLoader+# working around https://github.com/sphinx-doc/sphinx/issues/10400+from scrapy import Request, Spider # noqa: TC001from scrapy.utils.misc import load_object, walk_modules_iterfrom scrapy.utils.spider import iter_spider_classes@@ -18,7 +16,6 @@ if TYPE_CHECKING:# typing.Self requires Python 3.11from typing_extensions import Self- from scrapy import Request, Spiderfrom scrapy.settings import BaseSettings@@ -26,28 +23,31 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:"""Get SpiderLoader instance from settings"""cls_path = settings.get("SPIDER_LOADER_CLASS")loader_cls = load_object(cls_path)- verifyClass(ISpiderLoader, loader_cls)return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))class SpiderLoaderProtocol(Protocol):+ """Protocol for spider loader implementations.++ See :setting:`SPIDER_LOADER_CLASS`.+ """+@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:- """Return an instance of the class for the given settings"""+ """Return an instance of the class for the given settings."""def load(self, spider_name: str) -> type[Spider]:- """Return the Spider class for the given spider name. If the spider- name is not found, it must raise a KeyError."""+ """Return the spider class for the given spider name. If the spider+ name is not found, it must raise a :exc:`KeyError`."""def list(self) -> list[str]:"""Return a list with the names of all spiders available in the- project"""+ project."""def find_by_request(self, request: Request) -> __builtins__.list[str]:- """Return the list of spiders names that can handle the given request"""+ """Return the list of spiders names that can handle the given request."""-@implementer(ISpiderLoader)class SpiderLoader:"""SpiderLoader is a class which locates and loads spiders@@ -106,12 +106,18 @@ class SpiderLoader:@classmethoddef from_settings(cls, settings: BaseSettings) -> Self:+ """Create an instance of the class.++ It's called with the current project settings, and it loads the spiders+ found recursively in the modules of the :setting:`SPIDER_MODULES`+ setting.+ """return cls(settings)def load(self, spider_name: str) -> type[Spider]:- """- Return the Spider class for the given spider name. If the spider- name is not found, raise a KeyError.+ """Return the spider class for the given spider name.++ If the spider name is not found, raise a :exc:`KeyError`."""try:return self._spiders[spider_name]@@ -121,19 +127,19 @@ class SpiderLoader:def find_by_request(self, request: Request) -> list[str]:"""Return the list of spider names that can handle the given request.++ It will try to match the request's url against the domains of+ the spiders."""return [name for name, cls in self._spiders.items() if cls.handles_request(request)]def list(self) -> list[str]:- """- Return a list with the names of all spiders available in the project.- """+ """Return a list with the names of all spiders available in the project."""return list(self._spiders.keys())-@implementer(ISpiderLoader)class DummySpiderLoader:"""A dummy spider loader that does not load any spiders."""diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.pyindex 2f3649b43..9e35a38e2 100644--- a/tests/test_crawler_runners.py+++ b/tests/test_crawler_runners.py@@ -4,7 +4,6 @@ import loggingfrom typing import TYPE_CHECKINGimport pytest-from zope.interface.exceptions import MultipleInvalidfrom scrapy.crawler import (AsyncCrawlerProcess,@@ -14,7 +13,6 @@ from scrapy.crawler import (CrawlerRunner,CrawlerRunnerBase,)-from scrapy.settings import Settingsfrom scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_futurefrom scrapy.utils.spider import DefaultSpiderfrom scrapy.utils.test import get_reactor_settings@@ -33,15 +31,6 @@ class SpiderLoaderWithWrongInterface:class TestCrawlerRunner:- def test_spider_manager_verify_interface(self) -> None:- settings = Settings(- {- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,- }- )- with pytest.raises(MultipleInvalid):- CrawlerRunner(settings)-def test_crawler_runner_accepts_dict(self) -> None:runner = CrawlerRunner({"foo": "bar"})assert runner.settings["foo"] == "bar"@@ -53,15 +42,6 @@ class TestCrawlerRunner:class TestAsyncCrawlerRunner:- def test_spider_manager_verify_interface(self) -> None:- settings = Settings(- {- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,- }- )- with pytest.raises(MultipleInvalid):- AsyncCrawlerRunner(settings)-def test_crawler_runner_accepts_dict(self) -> None:runner = AsyncCrawlerRunner({"foo": "bar"})assert runner.settings["foo"] == "bar"diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 88070513f..3f659ee0b 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -18,7 +18,6 @@ import lxml.etreeimport pytestfrom testfixtures import LogCapturefrom w3lib.url import file_uri_to_path-from zope.interface import implementerimport scrapyfrom scrapy import Spider, signals@@ -29,7 +28,6 @@ from scrapy.extensions.feedexport import (FeedExporter,FeedSlot,FileFeedStorage,- IFeedStorage,apply_uri_params,)from scrapy.utils.python import to_unicode@@ -83,7 +81,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):raise OSError("Cannot store")-@implementer(IFeedStorage)class LogOnStoreFileStorage:"""This storage logs inside `store` method.@@ -1300,7 +1297,6 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_storage_file_no_postprocessing(self):- @implementer(IFeedStorage)class Storage:def __init__(self, uri, *, feed_options=None):pass@@ -1322,7 +1318,6 @@ class TestFeedExport(TestFeedExportBase):@coroutine_testasync def test_storage_file_postprocessing(self):- @implementer(IFeedStorage)class Storage:def __init__(self, uri, *, feed_options=None):passdiff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.pyindex 6e0536110..3b50cd492 100644--- a/tests/test_feedexport_batch.py+++ b/tests/test_feedexport_batch.py@@ -12,12 +12,11 @@ from urllib.parse import urljoinimport lxml.etreeimport pytestfrom packaging.version import Version-from zope.interface.verify import verifyObjectimport scrapyfrom scrapy import Spiderfrom scrapy.exceptions import NotConfigured-from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage+from scrapy.extensions.feedexport import FeedExporter, S3FeedStoragefrom scrapy.settings import Settingsfrom scrapy.utils.python import to_unicodefrom scrapy.utils.test import get_crawler@@ -436,9 +435,6 @@ class TestBatchDeliveries(TestFeedExportBase):},},}- crawler = get_crawler(settings_dict=settings)- storage = S3FeedStorage.from_crawler(crawler, uri)- verifyObject(IFeedStorage, storage)class TestSpider(scrapy.Spider):name = "testspider"diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.pyindex b7009be62..82e2597a5 100644--- a/tests/test_feedexport_storages.py+++ b/tests/test_feedexport_storages.py@@ -12,7 +12,6 @@ from urllib.parse import quoteimport pytestfrom testfixtures import LogCapturefrom w3lib.url import path_to_file_uri-from zope.interface.verify import verifyObjectimport scrapyfrom scrapy.extensions.feedexport import (@@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import (FileFeedStorage,FTPFeedStorage,GCSFeedStorage,- IFeedStorage,S3FeedStorage,StdoutFeedStorage,)@@ -55,11 +53,6 @@ class TestFileFeedStorage:finally:os.chdir(old_cwd)- def test_interface(self, tmp_path):- path = tmp_path / "file.txt"- st = FileFeedStorage(str(path))- verifyObject(IFeedStorage, st)-@staticmethoddef _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:storage = FileFeedStorage(str(path), feed_options=feed_options)@@ -115,7 +108,6 @@ class TestFTPFeedStorage:uri,feed_options=feed_options,)- verifyObject(IFeedStorage, storage)spider = self.get_test_spider()file = storage.open(spider)file.write(content)@@ -259,7 +251,6 @@ class TestS3FeedStorage:bucket = "mybucket"key = "export.csv"storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")- verifyObject(IFeedStorage, storage)file = mock.MagicMock()diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.pyindex 81e49bb75..81840f7e0 100644--- a/tests/test_spiderloader/__init__.py+++ b/tests/test_spiderloader/__init__.py@@ -5,14 +5,12 @@ from pathlib import Pathfrom unittest import mockimport pytest-from zope.interface.verify import verifyObject# ugly hack to avoid cyclic imports of scrapy.spiders when running this test# aloneimport scrapyfrom scrapy.crawler import CrawlerRunnerfrom scrapy.http import Request-from scrapy.interfaces import ISpiderLoaderfrom scrapy.settings import Settingsfrom scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader@@ -45,9 +43,6 @@ def spider_loader(spider_loader_env):class TestSpiderLoader:- def test_interface(self, spider_loader):- verifyObject(ISpiderLoader, spider_loader)-def test_list(self, spider_loader):assert set(spider_loader.list()) == {"spider1",