Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Remove non-Twisted uses of zope.interface (#7731)

Andrey Rakhmatullin394c2797f340
diff --git a/docs/news.rst b/docs/news.rst
index 63f0d69e4..4ebab3c6a 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,28 @@
Release notes
=============
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- The following runtime usage of zope.interface_ interfaces is removed:
+
+ - :class:`~scrapy.spiderloader.SpiderLoader` and
+ :class:`~scrapy.spiderloader.DummySpiderLoader` are no longer marked
+ as implementing the ``ISpiderLoader`` interface.
+
+ - :func:`~scrapy.spiderloader.get_spider_loader` no longer checks that the
+ configured spider loader implements the ``ISpiderLoader`` interface.
+
+ - :class:`~scrapy.extensions.feedexport.BlockingFeedStorage`,
+ :class:`~scrapy.extensions.feedexport.FileFeedStorage` and
+ :class:`~scrapy.extensions.feedexport.StdoutFeedStorage` are no longer
+ marked as implementing the ``IFeedStorage`` interface.
+
+ (:issue:`6585`, :issue:`7731`)
+
.. _release-2.17.0:
Scrapy 2.17.0 (2026-07-07)
@@ -2643,7 +2665,7 @@ Deprecation removals
(:issue:`6109`, :issue:`6116`)
- A custom class assigned to the :setting:`SPIDER_LOADER_CLASS` setting that
- does not implement the :class:`~scrapy.interfaces.ISpiderLoader` interface
+ does not implement the ``ISpiderLoader`` interface
will now raise a :exc:`zope.interface.verify.DoesNotImplement` exception at
run time. Non-compliant classes have been triggering a deprecation warning
since Scrapy 1.0.0.
diff --git a/docs/topics/api.rst b/docs/topics/api.rst
index 19082d9d7..7ff8d3464 100644
--- a/docs/topics/api.rst
+++ b/docs/topics/api.rst
@@ -172,46 +172,15 @@ SpiderLoader API
.. module:: scrapy.spiderloader
:synopsis: The spider loader
-.. class:: SpiderLoader
+Custom spider loaders can be employed by specifying their path in the
+:setting:`SPIDER_LOADER_CLASS` project setting. They must implement
+:class:`SpiderLoaderProtocol`.
- This class is in charge of retrieving and handling the spider classes
- defined across the project.
-
- Custom spider loaders can be employed by specifying their path in the
- :setting:`SPIDER_LOADER_CLASS` project setting. They must fully implement
- the :class:`scrapy.interfaces.ISpiderLoader` interface to guarantee an
- errorless execution.
-
- .. method:: from_settings(settings)
-
- This class method is used by Scrapy to create an instance of the class.
- It's called with the current project settings, and it loads the spiders
- found recursively in the modules of the :setting:`SPIDER_MODULES`
- setting.
-
- :param settings: project settings
- :type settings: :class:`~scrapy.settings.Settings` instance
-
- .. method:: load(spider_name)
-
- Get the Spider class with the given name. It'll look into the previously
- loaded spiders for a spider class with name ``spider_name`` and will raise
- a KeyError if not found.
-
- :param spider_name: spider class name
- :type spider_name: str
-
- .. method:: list()
-
- Get the names of the available spiders in the project.
-
- .. method:: find_by_request(request)
-
- List the spiders' names that can handle the given request. Will try to
- match the request's url against the domains of the spiders.
+.. autoclass:: SpiderLoaderProtocol
+ :members:
- :param request: queried request
- :type request: :class:`~scrapy.Request` instance
+.. autoclass:: SpiderLoader
+ :members:
.. autoclass:: DummySpiderLoader
diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
index a934cbbc7..ef948997d 100644
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -13,7 +13,6 @@ from twisted.internet.ssl import (
from twisted.web.client import BrowserLikePolicyForHTTPS
from twisted.web.iweb import IPolicyForHTTPS
from zope.interface.declarations import implementer
-from zope.interface.verify import verifyObject
from scrapy.core.downloader.tls import (
_TWISTED_VERSION_MAP,
@@ -232,7 +231,6 @@ class _AcceptableProtocolsContextFactory:
# all of this with _ScrapyClientContextFactory.acceptableProtocols.
def __init__(self, context_factory: Any, acceptable_protocols: list[bytes]):
- verifyObject(IPolicyForHTTPS, context_factory)
self._wrapped_context_factory: Any = context_factory
self._acceptable_protocols: list[bytes] = acceptable_protocols
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index 0d9b016a3..678a29e2e 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -22,7 +22,7 @@ from urllib.parse import unquote, urlparse
from twisted.internet.defer import Deferred, DeferredList
from w3lib.url import file_uri_to_path
-from zope.interface import Interface, implementer
+from zope.interface import Interface
from scrapy import Spider, signals
from scrapy.exceptions import NotConfigured, ScrapyDeprecationWarning
@@ -115,25 +115,18 @@ class ItemFilter:
return True # accept all items by default
-class IFeedStorage(Interface): # type: ignore[misc]
- """Interface that all Feed Storages must implement"""
-
+class _IFeedStorage(Interface): # type: ignore[misc] # pragma: no cover
# pylint: disable=no-self-argument
- def __init__(uri, *, feed_options=None): # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
- """Initialize the storage with the parameters given in the URI and the
- feed-specific options (see :setting:`FEEDS`)"""
+ def __init__(uri, *, feed_options=None): ... # type: ignore[no-untyped-def] # pylint: disable=super-init-not-called
- def open(spider): # type: ignore[no-untyped-def]
- """Open the storage for the given spider. It must return a file-like
- object that will be used for the exporters"""
+ def open(spider): ... # type: ignore[no-untyped-def]
- def store(file): # type: ignore[no-untyped-def]
- """Store the given file stream"""
+ def store(file): ... # type: ignore[no-untyped-def]
class FeedStorageProtocol(Protocol):
- """Reimplementation of ``IFeedStorage`` that can be used in type hints."""
+ """Protocol that all Feed Storages must follow."""
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
"""Initialize the storage with the parameters given in the URI and the
@@ -147,7 +140,6 @@ class FeedStorageProtocol(Protocol):
"""Store the given file stream"""
-@implementer(IFeedStorage)
class BlockingFeedStorage(ABC):
def open(self, spider: Spider) -> IO[bytes]:
path = spider.crawler.settings["FEED_TEMPDIR"]
@@ -164,7 +156,6 @@ class BlockingFeedStorage(ABC):
raise NotImplementedError
-@implementer(IFeedStorage)
class StdoutFeedStorage:
def __init__(
self,
@@ -191,7 +182,6 @@ class StdoutFeedStorage:
pass
-@implementer(IFeedStorage)
class FileFeedStorage:
def __init__(self, uri: str, *, feed_options: dict[str, Any] | None = None):
self.path: str = file_uri_to_path(uri) if uri.startswith("file:") else uri
@@ -758,3 +748,14 @@ class FeedExporter:
feed_options.get("item_filter", ItemFilter)
)
return item_filter_class(feed_options)
+
+
+def __getattr__(name: str) -> Any: # pragma: no cover
+ if name == "IFeedStorage":
+ warnings.warn(
+ "scrapy.extensions.feedexport.IFeedStorage is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+ return _IFeedStorage
+ raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
diff --git a/scrapy/interfaces.py b/scrapy/interfaces.py
index b4f1d9394..34d7bdd82 100644
--- a/scrapy/interfaces.py
+++ b/scrapy/interfaces.py
@@ -1,19 +1,23 @@
+# pragma: no file cover
# pylint: disable=no-method-argument,no-self-argument
+import warnings
from zope.interface import Interface
+from scrapy.exceptions import ScrapyDeprecationWarning
+
+warnings.warn(
+ "The scrapy.interfaces module is deprecated.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
+
class ISpiderLoader(Interface):
- def from_settings(settings):
- """Return an instance of the class for the given settings"""
+ def from_settings(settings): ...
- def load(spider_name):
- """Return the Spider class for the given spider name. If the spider
- name is not found, it must raise a KeyError."""
+ def load(spider_name): ...
- def list():
- """Return a list with the names of all spiders available in the
- project"""
+ def list(): ...
- def find_by_request(request):
- """Return the list of spiders names that can handle the given request"""
+ def find_by_request(request): ...
diff --git a/scrapy/spiderloader.py b/scrapy/spiderloader.py
index 8c980fd46..244b7338f 100644
--- a/scrapy/spiderloader.py
+++ b/scrapy/spiderloader.py
@@ -5,10 +5,8 @@ import warnings
from collections import defaultdict
from typing import TYPE_CHECKING, Protocol, cast
-from zope.interface import implementer
-from zope.interface.verify import verifyClass
-
-from scrapy.interfaces import ISpiderLoader
+# working around https://github.com/sphinx-doc/sphinx/issues/10400
+from scrapy import Request, Spider # noqa: TC001
from scrapy.utils.misc import load_object, walk_modules_iter
from scrapy.utils.spider import iter_spider_classes
@@ -18,7 +16,6 @@ if TYPE_CHECKING:
# typing.Self requires Python 3.11
from typing_extensions import Self
- from scrapy import Request, Spider
from scrapy.settings import BaseSettings
@@ -26,28 +23,31 @@ def get_spider_loader(settings: BaseSettings) -> SpiderLoaderProtocol:
"""Get SpiderLoader instance from settings"""
cls_path = settings.get("SPIDER_LOADER_CLASS")
loader_cls = load_object(cls_path)
- verifyClass(ISpiderLoader, loader_cls)
return cast("SpiderLoaderProtocol", loader_cls.from_settings(settings.frozencopy()))
class SpiderLoaderProtocol(Protocol):
+ """Protocol for spider loader implementations.
+
+ See :setting:`SPIDER_LOADER_CLASS`.
+ """
+
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
- """Return an instance of the class for the given settings"""
+ """Return an instance of the class for the given settings."""
def load(self, spider_name: str) -> type[Spider]:
- """Return the Spider class for the given spider name. If the spider
- name is not found, it must raise a KeyError."""
+ """Return the spider class for the given spider name. If the spider
+ name is not found, it must raise a :exc:`KeyError`."""
def list(self) -> list[str]:
"""Return a list with the names of all spiders available in the
- project"""
+ project."""
def find_by_request(self, request: Request) -> __builtins__.list[str]:
- """Return the list of spiders names that can handle the given request"""
+ """Return the list of spiders names that can handle the given request."""
-@implementer(ISpiderLoader)
class SpiderLoader:
"""
SpiderLoader is a class which locates and loads spiders
@@ -106,12 +106,18 @@ class SpiderLoader:
@classmethod
def from_settings(cls, settings: BaseSettings) -> Self:
+ """Create an instance of the class.
+
+ It's called with the current project settings, and it loads the spiders
+ found recursively in the modules of the :setting:`SPIDER_MODULES`
+ setting.
+ """
return cls(settings)
def load(self, spider_name: str) -> type[Spider]:
- """
- Return the Spider class for the given spider name. If the spider
- name is not found, raise a KeyError.
+ """Return the spider class for the given spider name.
+
+ If the spider name is not found, raise a :exc:`KeyError`.
"""
try:
return self._spiders[spider_name]
@@ -121,19 +127,19 @@ class SpiderLoader:
def find_by_request(self, request: Request) -> list[str]:
"""
Return the list of spider names that can handle the given request.
+
+ It will try to match the request's url against the domains of
+ the spiders.
"""
return [
name for name, cls in self._spiders.items() if cls.handles_request(request)
]
def list(self) -> list[str]:
- """
- Return a list with the names of all spiders available in the project.
- """
+ """Return a list with the names of all spiders available in the project."""
return list(self._spiders.keys())
-@implementer(ISpiderLoader)
class DummySpiderLoader:
"""A dummy spider loader that does not load any spiders."""
diff --git a/tests/test_crawler_runners.py b/tests/test_crawler_runners.py
index 2f3649b43..9e35a38e2 100644
--- a/tests/test_crawler_runners.py
+++ b/tests/test_crawler_runners.py
@@ -4,7 +4,6 @@ import logging
from typing import TYPE_CHECKING
import pytest
-from zope.interface.exceptions import MultipleInvalid
from scrapy.crawler import (
AsyncCrawlerProcess,
@@ -14,7 +13,6 @@ from scrapy.crawler import (
CrawlerRunner,
CrawlerRunnerBase,
)
-from scrapy.settings import Settings
from scrapy.utils.defer import ensure_awaitable, maybe_deferred_to_future
from scrapy.utils.spider import DefaultSpider
from scrapy.utils.test import get_reactor_settings
@@ -33,15 +31,6 @@ class SpiderLoaderWithWrongInterface:
class TestCrawlerRunner:
- def test_spider_manager_verify_interface(self) -> None:
- settings = Settings(
- {
- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
- }
- )
- with pytest.raises(MultipleInvalid):
- CrawlerRunner(settings)
-
def test_crawler_runner_accepts_dict(self) -> None:
runner = CrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
@@ -53,15 +42,6 @@ class TestCrawlerRunner:
class TestAsyncCrawlerRunner:
- def test_spider_manager_verify_interface(self) -> None:
- settings = Settings(
- {
- "SPIDER_LOADER_CLASS": SpiderLoaderWithWrongInterface,
- }
- )
- with pytest.raises(MultipleInvalid):
- AsyncCrawlerRunner(settings)
-
def test_crawler_runner_accepts_dict(self) -> None:
runner = AsyncCrawlerRunner({"foo": "bar"})
assert runner.settings["foo"] == "bar"
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 88070513f..3f659ee0b 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,7 +18,6 @@ import lxml.etree
import pytest
from testfixtures import LogCapture
from w3lib.url import file_uri_to_path
-from zope.interface import implementer
import scrapy
from scrapy import Spider, signals
@@ -29,7 +28,6 @@ from scrapy.extensions.feedexport import (
FeedExporter,
FeedSlot,
FileFeedStorage,
- IFeedStorage,
apply_uri_params,
)
from scrapy.utils.python import to_unicode
@@ -83,7 +81,6 @@ class FailingBlockingFeedStorage(DummyBlockingFeedStorage):
raise OSError("Cannot store")
-@implementer(IFeedStorage)
class LogOnStoreFileStorage:
"""
This storage logs inside `store` method.
@@ -1300,7 +1297,6 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_storage_file_no_postprocessing(self):
- @implementer(IFeedStorage)
class Storage:
def __init__(self, uri, *, feed_options=None):
pass
@@ -1322,7 +1318,6 @@ class TestFeedExport(TestFeedExportBase):
@coroutine_test
async def test_storage_file_postprocessing(self):
- @implementer(IFeedStorage)
class Storage:
def __init__(self, uri, *, feed_options=None):
pass
diff --git a/tests/test_feedexport_batch.py b/tests/test_feedexport_batch.py
index 6e0536110..3b50cd492 100644
--- a/tests/test_feedexport_batch.py
+++ b/tests/test_feedexport_batch.py
@@ -12,12 +12,11 @@ from urllib.parse import urljoin
import lxml.etree
import pytest
from packaging.version import Version
-from zope.interface.verify import verifyObject
import scrapy
from scrapy import Spider
from scrapy.exceptions import NotConfigured
-from scrapy.extensions.feedexport import FeedExporter, IFeedStorage, S3FeedStorage
+from scrapy.extensions.feedexport import FeedExporter, S3FeedStorage
from scrapy.settings import Settings
from scrapy.utils.python import to_unicode
from scrapy.utils.test import get_crawler
@@ -436,9 +435,6 @@ class TestBatchDeliveries(TestFeedExportBase):
},
},
}
- crawler = get_crawler(settings_dict=settings)
- storage = S3FeedStorage.from_crawler(crawler, uri)
- verifyObject(IFeedStorage, storage)
class TestSpider(scrapy.Spider):
name = "testspider"
diff --git a/tests/test_feedexport_storages.py b/tests/test_feedexport_storages.py
index b7009be62..82e2597a5 100644
--- a/tests/test_feedexport_storages.py
+++ b/tests/test_feedexport_storages.py
@@ -12,7 +12,6 @@ from urllib.parse import quote
import pytest
from testfixtures import LogCapture
from w3lib.url import path_to_file_uri
-from zope.interface.verify import verifyObject
import scrapy
from scrapy.extensions.feedexport import (
@@ -20,7 +19,6 @@ from scrapy.extensions.feedexport import (
FileFeedStorage,
FTPFeedStorage,
GCSFeedStorage,
- IFeedStorage,
S3FeedStorage,
StdoutFeedStorage,
)
@@ -55,11 +53,6 @@ class TestFileFeedStorage:
finally:
os.chdir(old_cwd)
- def test_interface(self, tmp_path):
- path = tmp_path / "file.txt"
- st = FileFeedStorage(str(path))
- verifyObject(IFeedStorage, st)
-
@staticmethod
def _store(path: Path, feed_options: dict[str, Any] | None = None) -> None:
storage = FileFeedStorage(str(path), feed_options=feed_options)
@@ -115,7 +108,6 @@ class TestFTPFeedStorage:
uri,
feed_options=feed_options,
)
- verifyObject(IFeedStorage, storage)
spider = self.get_test_spider()
file = storage.open(spider)
file.write(content)
@@ -259,7 +251,6 @@ class TestS3FeedStorage:
bucket = "mybucket"
key = "export.csv"
storage = S3FeedStorage.from_crawler(crawler, f"s3://{bucket}/{key}")
- verifyObject(IFeedStorage, storage)
file = mock.MagicMock()
diff --git a/tests/test_spiderloader/__init__.py b/tests/test_spiderloader/__init__.py
index 81e49bb75..81840f7e0 100644
--- a/tests/test_spiderloader/__init__.py
+++ b/tests/test_spiderloader/__init__.py
@@ -5,14 +5,12 @@ from pathlib import Path
from unittest import mock
import pytest
-from zope.interface.verify import verifyObject
# ugly hack to avoid cyclic imports of scrapy.spiders when running this test
# alone
import scrapy
from scrapy.crawler import CrawlerRunner
from scrapy.http import Request
-from scrapy.interfaces import ISpiderLoader
from scrapy.settings import Settings
from scrapy.spiderloader import DummySpiderLoader, SpiderLoader, get_spider_loader
@@ -45,9 +43,6 @@ def spider_loader(spider_loader_env):
class TestSpiderLoader:
- def test_interface(self, spider_loader):
- verifyObject(ISpiderLoader, spider_loader)
-
def test_list(self, spider_loader):
assert set(spider_loader.list()) == {
"spider1",