Scrapy, a fast high-level web crawling & scraping framework for Python.
Mark abstract base classes as such. (#6930)
Andrey Rakhmatullin3843091c5fa5
diff --git a/docs/news.rst b/docs/news.rstindex aaeac347f..9830eda64 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -15,6 +15,35 @@ Backward-incompatible changes``True`` when running Scrapy via :ref:`its command-line tool<topics-commands-crawlerprocess>` to avoid a reactor mismatch exception.+- The classes listed below are now :term:`abstract base classes <abstract+ base class>`. They cannot be instantiated directly and their subclasses+ need to override the abstract methods listed below to be able to be+ instantiated. If you previously instantiated these classes directly, you+ will now need to subclass them and provide trivial (e.g. empty)+ implementations for the abstract methods.++ - :class:`scrapy.commands.ScrapyCommand`++ - :meth:`~scrapy.commands.ScrapyCommand.run`++ - :meth:`~scrapy.commands.ScrapyCommand.short_desc`++ - :class:`scrapy.exporters.BaseItemExporter`++ - :meth:`~scrapy.exporters.BaseItemExporter.export_item`++ - :class:`scrapy.extensions.feedexport.BlockingFeedStorage`++ - :meth:`~scrapy.extensions.feedexport.BlockingFeedStorage._store_in_thread`++ - :class:`scrapy.middleware.MiddlewareManager`++ - :meth:`~scrapy.middleware.MiddlewareManager._get_mwlist_from_settings`++ - :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`++ - :meth:`~scrapy.spidermiddlewares.referer.ReferrerPolicy.referrer`+.. _release-2.13.2:diff --git a/docs/topics/spider-middleware.rst b/docs/topics/spider-middleware.rstindex 67178b8fd..2eb59fe44 100644--- a/docs/topics/spider-middleware.rst+++ b/docs/topics/spider-middleware.rst@@ -354,7 +354,7 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``Acceptable values for REFERRER_POLICY*************************************-- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``+- either a path to a :class:`scrapy.spidermiddlewares.referer.ReferrerPolicy`subclass — a custom policy or one of the built-in ones (see classes below),- or one or more comma-separated standard W3C-defined string values,- or the special ``"scrapy-default"``.@@ -373,6 +373,8 @@ String value Class name (as a string)`"unsafe-url"`_ :class:`scrapy.spidermiddlewares.referer.UnsafeUrlPolicy`======================================= ========================================================================+.. autoclass:: ReferrerPolicy+.. autoclass:: DefaultReferrerPolicy.. warning::Scrapy's default referrer policy — just like `"no-referrer-when-downgrade"`_,diff --git a/scrapy/commands/__init__.py b/scrapy/commands/__init__.pyindex 4ce070e6e..d5945f6f5 100644--- a/scrapy/commands/__init__.py+++ b/scrapy/commands/__init__.py@@ -7,6 +7,7 @@ from __future__ import annotationsimport argparseimport builtinsimport os+from abc import ABC, abstractmethodfrom pathlib import Pathfrom typing import TYPE_CHECKING, Any@@ -22,7 +23,7 @@ if TYPE_CHECKING:from scrapy.settings import Settings-class ScrapyCommand:+class ScrapyCommand(ABC):requires_project: bool = Falserequires_crawler_process: bool = Truecrawler_process: CrawlerProcessBase | None = None # set in scrapy.cmdline@@ -46,6 +47,7 @@ class ScrapyCommand:"""return ""+ @abstractmethoddef short_desc(self) -> str:"""A short description of the command@@ -128,6 +130,7 @@ class ScrapyCommand:if opts.pdb:failure.startDebugMode()+ @abstractmethoddef run(self, args: list[str], opts: argparse.Namespace) -> None:"""Entry point for running commandsdiff --git a/scrapy/exporters.py b/scrapy/exporters.pyindex 0a641752e..e18f1e6ed 100644--- a/scrapy/exporters.py+++ b/scrapy/exporters.py@@ -8,6 +8,7 @@ import csvimport marshalimport pickleimport pprint+from abc import ABC, abstractmethodfrom collections.abc import Callable, Iterable, Mappingfrom io import BytesIO, TextIOWrapperfrom typing import TYPE_CHECKING, Any@@ -35,7 +36,7 @@ __all__ = []-class BaseItemExporter:+class BaseItemExporter(ABC):def __init__(self, *, dont_fail: bool = False, **kwargs: Any):self._kwargs: dict[str, Any] = kwargsself._configure(kwargs, dont_fail=dont_fail)@@ -54,6 +55,7 @@ class BaseItemExporter:if not dont_fail and options:raise TypeError(f"Unexpected options: {', '.join(options.keys())}")+ @abstractmethoddef export_item(self, item: Any) -> None:raise NotImplementedError@@ -63,10 +65,10 @@ class BaseItemExporter:serializer: Callable[[Any], Any] = field.get("serializer", lambda x: x)return serializer(value)- def start_exporting(self) -> None:+ def start_exporting(self) -> None: # noqa: B027pass- def finish_exporting(self) -> None:+ def finish_exporting(self) -> None: # noqa: B027passdef _get_serialized_fields(diff --git a/scrapy/extensions/feedexport.py b/scrapy/extensions/feedexport.pyindex f7bf50a5c..557ad99e1 100644--- a/scrapy/extensions/feedexport.py+++ b/scrapy/extensions/feedexport.py@@ -11,6 +11,7 @@ import loggingimport reimport sysimport warnings+from abc import ABC, abstractmethodfrom collections.abc import Callablefrom datetime import datetime, timezonefrom pathlib import Path, PureWindowsPath@@ -140,7 +141,7 @@ class FeedStorageProtocol(Protocol):@implementer(IFeedStorage)-class BlockingFeedStorage:+class BlockingFeedStorage(ABC):def open(self, spider: Spider) -> IO[bytes]:path = spider.crawler.settings["FEED_TEMPDIR"]if path and not Path(path).is_dir():@@ -151,6 +152,7 @@ class BlockingFeedStorage:def store(self, file: IO[bytes]) -> Deferred[None] | None:return deferToThread(self._store_in_thread, file)+ @abstractmethoddef _store_in_thread(self, file: IO[bytes]) -> None:raise NotImplementedErrordiff --git a/scrapy/middleware.py b/scrapy/middleware.pyindex 2b67dcd21..25f2aa199 100644--- a/scrapy/middleware.py+++ b/scrapy/middleware.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport loggingimport pprintimport warnings+from abc import ABC, abstractmethodfrom collections import defaultdict, dequefrom typing import TYPE_CHECKING, Any, TypeVar, cast@@ -32,10 +33,10 @@ _T = TypeVar("_T")_T2 = TypeVar("_T2")-class MiddlewareManager:+class MiddlewareManager(ABC):"""Base class for implementing middleware managers"""- component_name = "foo middleware"+ component_name: strdef __init__(self, *middlewares: Any) -> None:self.middlewares = middlewares@@ -48,6 +49,7 @@ class MiddlewareManager:self._add_middleware(mw)@classmethod+ @abstractmethoddef _get_mwlist_from_settings(cls, settings: Settings) -> list[Any]:raise NotImplementedErrordiff --git a/scrapy/spidermiddlewares/referer.py b/scrapy/spidermiddlewares/referer.pyindex f5d406c13..1d4d1527b 100644--- a/scrapy/spidermiddlewares/referer.py+++ b/scrapy/spidermiddlewares/referer.py@@ -6,6 +6,7 @@ originated it.from __future__ import annotationsimport warnings+from abc import ABC, abstractmethodfrom typing import TYPE_CHECKING, castfrom urllib.parse import urlparse@@ -45,10 +46,13 @@ POLICY_UNSAFE_URL = "unsafe-url"POLICY_SCRAPY_DEFAULT = "scrapy-default"-class ReferrerPolicy:+class ReferrerPolicy(ABC):+ """Abstract base class for referrer policies."""+NOREFERRER_SCHEMES: tuple[str, ...] = LOCAL_SCHEMESname: str+ @abstractmethoddef referrer(self, response_url: str, request_url: str) -> str | None:raise NotImplementedErrordiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 851c92db4..1b36fbb31 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -25,9 +25,17 @@ if TYPE_CHECKING:import os+class EmptyCommand(ScrapyCommand):+ def short_desc(self) -> str:+ return ""++ def run(self, args: list[str], opts: argparse.Namespace) -> None:+ pass++class TestCommandSettings:def setup_method(self):- self.command = ScrapyCommand()+ self.command = EmptyCommand()self.command.settings = Settings()self.parser = argparse.ArgumentParser(formatter_class=ScrapyHelpFormatter, conflict_handler="resolve"diff --git a/tests/test_exporters.py b/tests/test_exporters.pyindex 05e8865bc..5719d5bb0 100644--- a/tests/test_exporters.py+++ b/tests/test_exporters.py@@ -4,6 +4,7 @@ import marshalimport pickleimport reimport tempfile+from abc import ABC, abstractmethodfrom datetime import datetimefrom io import BytesIOfrom typing import Any@@ -53,7 +54,7 @@ class CustomFieldDataclass:age: int = dataclasses.field(metadata={"serializer": custom_serializer})-class TestBaseItemExporter:+class TestBaseItemExporter(ABC):item_class: type = MyItemcustom_field_item_class: type = CustomFieldItem@@ -62,10 +63,11 @@ class TestBaseItemExporter:self.output = BytesIO()self.ie = self._get_exporter()- def _get_exporter(self, **kwargs):- return BaseItemExporter(**kwargs)+ @abstractmethod+ def _get_exporter(self, **kwargs) -> BaseItemExporter:+ raise NotImplementedError- def _check_output(self):+ def _check_output(self): # noqa: B027passdef _assert_expected_item(self, exported_dict):@@ -83,11 +85,7 @@ class TestBaseItemExporter:def assertItemExportWorks(self, item):self.ie.start_exporting()- try:- self.ie.export_item(item)- except NotImplementedError:- if self.ie.__class__ is not BaseItemExporter:- raise+ self.ie.export_item(item)self.ie.finish_exporting()# Delete the item exporter object, so that if it causes the output# file handle to be closed, which should not be the case, follow-up@@ -132,11 +130,6 @@ class TestBaseItemExporter:assert ie.serialize_field(a.get_field_meta("age"), "age", a["age"]) == "24"-class TestBaseItemExporterDataclass(TestBaseItemExporter):- item_class = MyDataClass- custom_field_item_class = CustomFieldDataclass--class TestPythonItemExporter(TestBaseItemExporter):def _get_exporter(self, **kwargs):return PythonItemExporter(**kwargs)@@ -670,6 +663,9 @@ class TestCustomExporterItem:return str(int(value) + 1)return super().serialize_field(field, name, value)+ def export_item(self, item: Any) -> None:+ pass+i = self.item_class(name="John", age="22")a = ItemAdapter(i)ie = CustomItemExporter()diff --git a/tests/test_feedexport.py b/tests/test_feedexport.pyindex 7073d5a35..ce1b7952e 100644--- a/tests/test_feedexport.py+++ b/tests/test_feedexport.py@@ -18,7 +18,7 @@ from io import BytesIOfrom logging import getLoggerfrom pathlib import Pathfrom string import ascii_letters, digits-from typing import TYPE_CHECKING, Any+from typing import IO, TYPE_CHECKING, Anyfrom unittest import mockfrom urllib.parse import quote, urljoinfrom urllib.request import pathname2url@@ -236,6 +236,11 @@ class TestFTPFeedStorage(unittest.TestCase):assert st.password == string.punctuation+class MyBlockingFeedStorage(BlockingFeedStorage):+ def _store_in_thread(self, file: IO[bytes]) -> None:+ return++class TestBlockingFeedStorage:def get_test_spider(self, settings=None):class TestSpider(scrapy.Spider):@@ -245,14 +250,14 @@ class TestBlockingFeedStorage:return TestSpider.from_crawler(crawler)def test_default_temp_dir(self):- b = BlockingFeedStorage()+ b = MyBlockingFeedStorage()storage_file = b.open(self.get_test_spider())storage_dir = Path(storage_file.name).parentassert str(storage_dir) == tempfile.gettempdir()def test_temp_file(self, tmp_path):- b = BlockingFeedStorage()+ b = MyBlockingFeedStorage()spider = self.get_test_spider({"FEED_TEMPDIR": str(tmp_path)})storage_file = b.open(spider)@@ -260,7 +265,7 @@ class TestBlockingFeedStorage:assert storage_dir == tmp_pathdef test_invalid_folder(self, tmp_path):- b = BlockingFeedStorage()+ b = MyBlockingFeedStorage()invalid_path = tmp_path / "invalid_path"spider = self.get_test_spider({"FEED_TEMPDIR": str(invalid_path)})diff --git a/tests/test_middleware.py b/tests/test_middleware.pyindex d004d4d93..3dd1334b2 100644--- a/tests/test_middleware.py+++ b/tests/test_middleware.py@@ -39,6 +39,8 @@ class MOff:class MyMiddlewareManager(MiddlewareManager):+ component_name = "my"+@classmethoddef _get_mwlist_from_settings(cls, settings):return [M1, MOff, M3]@@ -65,7 +67,7 @@ class TestMiddlewareManager:def test_enabled(self):m1, m2, m3 = M1(), M2(), M3()- mwman = MiddlewareManager(m1, m2, m3)+ mwman = MyMiddlewareManager(m1, m2, m3)assert mwman.middlewares == (m1, m2, m3)def test_enabled_from_settings(self):