Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Mark abstract base classes as such. (#6930)

Andrey Rakhmatullin3843091c5fa5
diff --git a/docs/news.rst b/docs/news.rst
index aaeac347f..9830eda64 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -15,6 +15,35 @@ Backward-incompatible changes
``True`` when running Scrapy via :ref:`its command-line tool
<topics-commands-crawlerprocess>` to avoid a reactor mismatch exception.
+- The classes listed below are now :term:`abstract base classes <abstract
+ base class>`. They cannot be instantiated directly and their subclasses
+ need to override the abstract methods listed below to be able to be
+ instantiated. If you previously instantiated these classes directly, you
+ will now need to subclass them and provide trivial (e.g. empty)
+ implementations for the abstract methods.
+
+ - :class:`scrapy.commands.ScrapyCommand`
+
+ - :meth:`~scrapy.commands.ScrapyCommand.run`
+
+ - :meth:`~scrapy.commands.ScrapyCommand.short_desc`
+
+ - :class:`scrapy.exporters.BaseItemExporter`
+
+ - :meth:`~scrapy.exporters.BaseItemExporter.export_item`
+
+ - :class:`scrapy.extensions.feedexport.BlockingFeedStorage`
+
+ - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread`
+
+ - :class:`scrapy.middleware.MiddlewareManager`
+
+ - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings`
+
+ - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`
+
+ - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`
+
.. _release-2.13.2:
diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rst
index 67178b8fd..2eb59fe44 100644
--- a/docs/topics/spider-middleware.rst
+++ b/docs/topics/spider-middleware.rst
@@ -354,7 +354,7 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
Acceptable values for REFERRER_POLICY
*************************************
-- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
+- either a path to a :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`
subclass — a custom policy or one of the built-in ones (see classes below),
- or one or more comma-separated standard W3C-defined string values,
- or the special ``"scrapy-default"``.
@@ -373,6 +373,8 @@ String value Class name (as a string)
`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`
======================================= ========================================================================
+.. autoclass:: ReferrerPolicy
+
.. autoclass:: DefaultReferrerPolicy
.. warning::
Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,
diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.py
index 4ce070e6e..d5945f6f5 100644
--- a/scrapy/commands/__init__.py
+++ b/scrapy/commands/__init__.py
@@ -7,6 +7,7 @@ from __future__ import annotations
import argparse
import builtins
import os
+from abc import ABC, abstractmethod
from pathlib import Path
from typing import TYPE_CHECKING, Any
@@ -22,7 +23,7 @@ if TYPE_CHECKING:
from scrapy.settings import Settings
-class ScrapyCommand:
+class ScrapyCommand(ABC):
requires_project: bool = False
requires_crawler_process: bool = True
crawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline
@@ -46,6 +47,7 @@ class ScrapyCommand:
"""
return ""
+ @abstractmethod
def short_desc(self) -> str:
"""
A short description of the command
@@ -128,6 +130,7 @@ class ScrapyCommand:
if opts.pdb:
failure.startDebugMode()
+ @abstractmethod
def run(self, args: list[str], opts: argparse.Namespace) -> None:
"""
Entry point for running commands
diff --git a/scrapy/exporters.py b/scrapy/exporters.py
index 0a641752e..e18f1e6ed 100644
--- a/scrapy/exporters.py
+++ b/scrapy/exporters.py
@@ -8,6 +8,7 @@ import csv
import marshal
import pickle
import pprint
+from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable, Mapping
from io import BytesIO, TextIOWrapper
from typing import TYPE_CHECKING, Any
@@ -35,7 +36,7 @@ __all__ = [
]
-class BaseItemExporter:
+class BaseItemExporter(ABC):
def __init__(self, *, dont_fail: bool = False, **kwargs: Any):
self._kwargs: dict[str, Any] = kwargs
self._configure(kwargs, dont_fail=dont_fail)
@@ -54,6 +55,7 @@ class BaseItemExporter:
if not dont_fail and options:
raise TypeError(f"Unexpected options: {', '.join(options.keys())}")
+ @abstractmethod
def export_item(self, item: Any) -> None:
raise NotImplementedError
@@ -63,10 +65,10 @@ class BaseItemExporter:
serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)
return serializer(value)
- def start_exporting(self) -> None:
+ def start_exporting(self) -> None: # noqa: B027
pass
- def finish_exporting(self) -> None:
+ def finish_exporting(self) -> None: # noqa: B027
pass
def _get_serialized_fields(
diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.py
index f7bf50a5c..557ad99e1 100644
--- a/scrapy/extensions/feedexport.py
+++ b/scrapy/extensions/feedexport.py
@@ -11,6 +11,7 @@ import logging
import re
import sys
import warnings
+from abc import ABC, abstractmethod
from collections.abc import Callable
from datetime import datetime, timezone
from pathlib import Path, PureWindowsPath
@@ -140,7 +141,7 @@ class FeedStorageProtocol(Protocol):
@implementer(IFeedStorage)
-class BlockingFeedStorage:
+class BlockingFeedStorage(ABC):
def open(self, spider: Spider) -> IO[bytes]:
path = spider.crawler.settings["FEED_TEMPDIR"]
if path and not Path(path).is_dir():
@@ -151,6 +152,7 @@ class BlockingFeedStorage:
def store(self, file: IO[bytes]) -> Deferred[None] | None:
return deferToThread(self._store_in_thread, file)
+ @abstractmethod
def _store_in_thread(self, file: IO[bytes]) -> None:
raise NotImplementedError
diff --git a/scrapy/middleware.py b/scrapy/middleware.py
index 2b67dcd21..25f2aa199 100644
--- a/scrapy/middleware.py
+++ b/scrapy/middleware.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
import pprint
import warnings
+from abc import ABC, abstractmethod
from collections import defaultdict, deque
from typing import TYPE_CHECKING, Any, TypeVar, cast
@@ -32,10 +33,10 @@ _T = TypeVar("_T")
_T2 = TypeVar("_T2")
-class MiddlewareManager:
+class MiddlewareManager(ABC):
"""Base class for implementing middleware managers"""
- component_name = "foo middleware"
+ component_name: str
def __init__(self, *middlewares: Any) -> None:
self.middlewares = middlewares
@@ -48,6 +49,7 @@ class MiddlewareManager:
self._add_middleware(mw)
@classmethod
+ @abstractmethod
def _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:
raise NotImplementedError
diff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.py
index f5d406c13..1d4d1527b 100644
--- a/scrapy/spidermiddlewares/referer.py
+++ b/scrapy/spidermiddlewares/referer.py
@@ -6,6 +6,7 @@ originated it.
from __future__ import annotations
import warnings
+from abc import ABC, abstractmethod
from typing import TYPE_CHECKING, cast
from urllib.parse import urlparse
@@ -45,10 +46,13 @@ POLICY_UNSAFE_URL = "unsafe-url"
POLICY_SCRAPY_DEFAULT = "scrapy-default"
-class ReferrerPolicy:
+class ReferrerPolicy(ABC):
+ """Abstract base class for referrer policies."""
+
NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMES
name: str
+ @abstractmethod
def referrer(self, response_url: str, request_url: str) -> str | None:
raise NotImplementedError
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 851c92db4..1b36fbb31 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -25,9 +25,17 @@ if TYPE_CHECKING:
import os
+class EmptyCommand(ScrapyCommand):
+ def short_desc(self) -> str:
+ return ""
+
+ def run(self, args: list[str], opts: argparse.Namespace) -> None:
+ pass
+
+
class TestCommandSettings:
def setup_method(self):
- self.command = ScrapyCommand()
+ self.command = EmptyCommand()
self.command.settings = Settings()
self.parser = argparse.ArgumentParser(
formatter_class=ScrapyHelpFormatter, conflict_handler="resolve"
diff --git a/tests/test_exporters.py b/tests/test_exporters.py
index 05e8865bc..5719d5bb0 100644
--- a/tests/test_exporters.py
+++ b/tests/test_exporters.py
@@ -4,6 +4,7 @@ import marshal
import pickle
import re
import tempfile
+from abc import ABC, abstractmethod
from datetime import datetime
from io import BytesIO
from typing import Any
@@ -53,7 +54,7 @@ class CustomFieldDataclass:
age: int = dataclasses.field(metadata={"serializer": custom_serializer})
-class TestBaseItemExporter:
+class TestBaseItemExporter(ABC):
item_class: type = MyItem
custom_field_item_class: type = CustomFieldItem
@@ -62,10 +63,11 @@ class TestBaseItemExporter:
self.output = BytesIO()
self.ie = self._get_exporter()
- def _get_exporter(self, **kwargs):
- return BaseItemExporter(**kwargs)
+ @abstractmethod
+ def _get_exporter(self, **kwargs) -> BaseItemExporter:
+ raise NotImplementedError
- def _check_output(self):
+ def _check_output(self): # noqa: B027
pass
def _assert_expected_item(self, exported_dict):
@@ -83,11 +85,7 @@ class TestBaseItemExporter:
def assertItemExportWorks(self, item):
self.ie.start_exporting()
- try:
- self.ie.export_item(item)
- except NotImplementedError:
- if self.ie.__class__ is not BaseItemExporter:
- raise
+ self.ie.export_item(item)
self.ie.finish_exporting()
# Delete the item exporter object, so that if it causes the output
# file handle to be closed, which should not be the case, follow-up
@@ -132,11 +130,6 @@ class TestBaseItemExporter:
assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24"
-class TestBaseItemExporterDataclass(TestBaseItemExporter):
- item_class = MyDataClass
- custom_field_item_class = CustomFieldDataclass
-
-
class TestPythonItemExporter(TestBaseItemExporter):
def _get_exporter(self, **kwargs):
return PythonItemExporter(**kwargs)
@@ -670,6 +663,9 @@ class TestCustomExporterItem:
return str(int(value) + 1)
return super().serialize_field(field, name, value)
+ def export_item(self, item: Any) -> None:
+ pass
+
i = self.item_class(name="John", age="22")
a = ItemAdapter(i)
ie = CustomItemExporter()
diff --git a/tests/test_feedexport.py b/tests/test_feedexport.py
index 7073d5a35..ce1b7952e 100644
--- a/tests/test_feedexport.py
+++ b/tests/test_feedexport.py
@@ -18,7 +18,7 @@ from io import BytesIO
from logging import getLogger
from pathlib import Path
from string import ascii_letters, digits
-from typing import TYPE_CHECKING, Any
+from typing import IO, TYPE_CHECKING, Any
from unittest import mock
from urllib.parse import quote, urljoin
from urllib.request import pathname2url
@@ -236,6 +236,11 @@ class TestFTPFeedStorage(unittest.TestCase):
assert st.password == string.punctuation
+class MyBlockingFeedStorage(BlockingFeedStorage):
+ def _store_in_thread(self, file: IO[bytes]) -> None:
+ return
+
+
class TestBlockingFeedStorage:
def get_test_spider(self, settings=None):
class TestSpider(scrapy.Spider):
@@ -245,14 +250,14 @@ class TestBlockingFeedStorage:
return TestSpider.from_crawler(crawler)
def test_default_temp_dir(self):
- b = BlockingFeedStorage()
+ b = MyBlockingFeedStorage()
storage_file = b.open(self.get_test_spider())
storage_dir = Path(storage_file.name).parent
assert str(storage_dir) == tempfile.gettempdir()
def test_temp_file(self, tmp_path):
- b = BlockingFeedStorage()
+ b = MyBlockingFeedStorage()
spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)})
storage_file = b.open(spider)
@@ -260,7 +265,7 @@ class TestBlockingFeedStorage:
assert storage_dir == tmp_path
def test_invalid_folder(self, tmp_path):
- b = BlockingFeedStorage()
+ b = MyBlockingFeedStorage()
invalid_path = tmp_path / "invalid_path"
spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})
diff --git a/tests/test_middleware.py b/tests/test_middleware.py
index d004d4d93..3dd1334b2 100644
--- a/tests/test_middleware.py
+++ b/tests/test_middleware.py
@@ -39,6 +39,8 @@ class MOff:
class MyMiddlewareManager(MiddlewareManager):
+ component_name = "my"
+
@classmethod
def _get_mwlist_from_settings(cls, settings):
return [M1, MOff, M3]
@@ -65,7 +67,7 @@ class TestMiddlewareManager:
def test_enabled(self):
m1, m2, m3 = M1(), M2(), M3()
- mwman = MiddlewareManager(m1, m2, m3)
+ mwman = MyMiddlewareManager(m1, m2, m3)
assert mwman.middlewares == (m1, m2, m3)
def test_enabled_from_settings(self):