Scrapy, a fast high-level web crawling & scraping framework for Python.
Make NO_CALLBACK a callable
Adrián Chavesf03b47db05e6
diff --git a/scrapy/http/request/__init__.py b/scrapy/http/request/__init__.pyindex 068a4baa3..de13cf264 100644--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@ -5,10 +5,8 @@ requests in Scrapy.See documentation in docs/topics/request-response.rst"""import inspect-from enum import Enumfrom typing import Callable, List, Optional, Tuple, Type, TypeVar, Union-from typing_extensions import Finalfrom w3lib.url import safe_url_stringimport scrapy@@ -23,21 +21,22 @@ from scrapy.utils.url import escape_ajaxRequestTypeVar = TypeVar("RequestTypeVar", bound="Request")-# https://github.com/python/typing/issues/689#issuecomment-561425237-class NoCallbackType(Enum):- NO_CALLBACK = 0+def NO_CALLBACK(*args, **kwargs):+ """When assigned to the ``callback`` parameter of+ :class:`~scrapy.http.Request`, it indicates that the request is not meant+ to have a spider callback at all.--#: When assigned to the ``callback`` parameter of-#: :class:`~scrapy.http.Request`, it indicates that the request is not meant to-#: have a spider callback at all.-#:-#: This value should be used by :ref:`components <topics-components>`-#: that create and handle their own requests, e.g. through-#: :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download-#: middlewares handling such requests can treat them differently from requests-#: intended for the :meth:`~scrapy.Spider.parse` callback.-NO_CALLBACK: Final = NoCallbackType.NO_CALLBACK+ This value should be used by :ref:`components <topics-components>` that+ create and handle their own requests, e.g. through+ :meth:`scrapy.core.engine.ExecutionEngine.download`, so that download+ middlewares handling such requests can treat them differently from requests+ intended for the :meth:`~scrapy.Spider.parse` callback.+ """+ raise RuntimeError(+ "The NO_CALLBACK callback has been called. This is a special callback "+ "value intended for requests whose callback is never meant to be "+ "called."+ )class Request(object_ref):@@ -67,8 +66,6 @@ class Request(object_ref):Currently used by :meth:`Request.replace`, :meth:`Request.to_dict` and:func:`~scrapy.utils.request.request_from_dict`."""- callback: Union[None, NoCallbackType, Callable]- errback: Optional[Callable]def __init__(self,@@ -94,8 +91,14 @@ class Request(object_ref):raise TypeError(f"Request priority not an integer: {priority!r}")self.priority = priority- self._set_xback("callback", callback)- self._set_xback("errback", errback)+ if not (callable(callback) or callback is None):+ raise TypeError(+ f"callback must be a callable, got {type(callback).__name__}"+ )+ if not (callable(errback) or errback is None):+ raise TypeError(f"errback must be a callable, got {type(errback).__name__}")+ self.callback = callback+ self.errback = errbackself.cookies = cookies or {}self.headers = Headers(headers or {}, encoding=encoding)@@ -105,15 +108,6 @@ class Request(object_ref):self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else Noneself.flags = [] if flags is None else list(flags)- def _set_xback(self, name: str, value: Optional[Callable]) -> None:- if not (- callable(value)- or value is None- or (name == "callback" and value is NO_CALLBACK)- ):- raise TypeError(f"{name} must be a callable, got {type(value).__name__}")- setattr(self, name, value)-@propertydef cb_kwargs(self) -> dict:if self._cb_kwargs is None:diff --git a/setup.py b/setup.pyindex 9150dac0b..f53334d4e 100644--- a/setup.py+++ b/setup.py@@ -34,7 +34,6 @@ install_requires = ["packaging","tldextract","lxml>=4.3.0",- "typing-extensions>=3.10.0.0",]extras_require = {}cpython_dependencies = [diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex 233a5f0b2..e800f427f 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -314,8 +314,10 @@ class RequestTest(unittest.TestCase):r5 = self.request_class(url="http://example.com",callback=NO_CALLBACK,+ errback=NO_CALLBACK,)self.assertIs(r5.callback, NO_CALLBACK)+ self.assertIs(r5.errback, NO_CALLBACK)def test_callback_and_errback_type(self):with self.assertRaises(TypeError):@@ -328,8 +330,10 @@ class RequestTest(unittest.TestCase):callback="a_function",errback="a_function",)- with self.assertRaises(TypeError):- self.request_class("http://example.com", errback=NO_CALLBACK)++ def test_no_callback(self):+ with self.assertRaises(RuntimeError):+ NO_CALLBACK()def test_from_curl(self):# Note: more curated tests regarding curl conversion are in