Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge remote-tracking branch 'scrapy/master' into no-callback
Adrián Chaves9272c4af0c60
diff --cc scrapy/downloadermiddlewares/robotstxt.pyindex 67e14b7b5,326c35290..23b5924e1--- a/scrapy/downloadermiddlewares/robotstxt.py+++ b/scrapy/downloadermiddlewares/robotstxt.py@@@ -66,8 -68,7 +69,8 @@@ class RobotsTxtMiddlewarerobotsreq = Request(robotsurl,priority=self.DOWNLOAD_PRIORITY,- meta={'dont_obey_robotstxt': True},+ meta={"dont_obey_robotstxt": True},+ callback=NO_CALLBACK,)dfd = self.crawler.engine.download(robotsreq)dfd.addCallback(self._parse_robots, netloc, spider)diff --cc scrapy/http/request/__init__.pyindex 936afb007,e290f2143..a78ba5115--- a/scrapy/http/request/__init__.py+++ b/scrapy/http/request/__init__.py@@@ -95,15 -90,6 +105,15 @@@ class Request(object_ref)self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else Noneself.flags = [] if flags is None else list(flags)+ def _set_xback(self, name: str, value: Optional[Callable]) -> None:+ if (+ value is not None+ and (name != "callback" or value is not NO_CALLBACK)+ and not callable(value)+ ):- raise TypeError(f'{name} must be a callable, got {type(value).__name__}')++ raise TypeError(f"{name} must be a callable, got {type(value).__name__}")+ setattr(self, name, value)+@propertydef cb_kwargs(self) -> dict:if self._cb_kwargs is None:diff --cc setup.pyindex 049c4f9a6,f53334d4e..9150dac0b--- a/setup.py+++ b/setup.py@@@ -18,23 -18,22 +18,23 @@@ def has_environment_marker_platform_impinstall_requires = [- 'Twisted>=18.9.0',- 'cryptography>=3.4.6',- 'cssselect>=0.9.1',- 'itemloaders>=1.0.1',- 'parsel>=1.5.0',- 'pyOpenSSL>=21.0.0',- 'queuelib>=1.4.2',- 'service_identity>=18.1.0',- 'w3lib>=1.17.0',- 'zope.interface>=5.1.0',- 'protego>=0.1.15',- 'itemadapter>=0.1.0',- 'setuptools',- 'packaging',- 'tldextract',- 'lxml>=4.3.0',- 'typing-extensions>=3.10.0.0',+ "Twisted>=18.9.0",+ "cryptography>=3.4.6",+ "cssselect>=0.9.1",+ "itemloaders>=1.0.1",+ "parsel>=1.5.0",+ "pyOpenSSL>=21.0.0",+ "queuelib>=1.4.2",+ "service_identity>=18.1.0",+ "w3lib>=1.17.0",+ "zope.interface>=5.1.0",+ "protego>=0.1.15",+ "itemadapter>=0.1.0",+ "setuptools",+ "packaging",+ "tldextract",+ "lxml>=4.3.0",++ "typing-extensions>=3.10.0.0",]extras_require = {}cpython_dependencies = [diff --cc tests/test_downloadermiddleware_robotstxt.pyindex 71d53ff1a,ac08c6006..953b61626--- a/tests/test_downloadermiddleware_robotstxt.py+++ b/tests/test_downloadermiddleware_robotstxt.py@@@ -4,11 -4,12 +4,13 @@@ from twisted.internet import reactor, efrom twisted.internet.defer import Deferred, DeferredList, maybeDeferredfrom twisted.python import failurefrom twisted.trial import unittest- from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,- logger as mw_module_logger)+ from scrapy.downloadermiddlewares.robotstxt import (+ RobotsTxtMiddleware,+ logger as mw_module_logger,+ )from scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request, Response, TextResponse+from scrapy.http.request import NO_CALLBACKfrom scrapy.settings import Settingsfrom tests.test_robotstxt_interface import rerp_available, reppy_available@@@ -52,14 -55,21 +56,22 @@@ Disallow: /some/randome/page.htmdef test_robotstxt(self):middleware = RobotsTxtMiddleware(self._get_successful_crawler())- return DeferredList([- self.assertNotIgnored(Request('http://site.local/allowed'), middleware),- maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),- self.assertIgnored(Request('http://site.local/admin/main'), middleware),- self.assertIgnored(Request('http://site.local/static/'), middleware),- self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),- self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)- ], fireOnOneErrback=True)+ return DeferredList(+ [+ self.assertNotIgnored(Request("http://site.local/allowed"), middleware),++ maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),+ self.assertIgnored(Request("http://site.local/admin/main"), middleware),+ self.assertIgnored(Request("http://site.local/static/"), middleware),+ self.assertIgnored(+ Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"),+ middleware,+ ),+ self.assertIgnored(+ Request("http://site.local/wiki/Käyttäjä:"), middleware+ ),+ ],+ fireOnOneErrback=True,+ )def test_robotstxt_ready_parser(self):middleware = RobotsTxtMiddleware(self._get_successful_crawler())@@@ -182,15 -222,10 +224,16 @@@def assertIgnored(self, request, middleware):spider = None # not actually used- return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),- IgnoreRequest)+ return self.assertFailure(+ maybeDeferred(middleware.process_request, request, spider), IgnoreRequest+ )+ def assertRobotsTxtRequested(self, base_url):+ calls = self.crawler.engine.download.call_args_list+ request = calls[0][0][0]+ self.assertEqual(request.url, f"{base_url}/robots.txt")+ self.assertEqual(request.callback, NO_CALLBACK)+class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):if not rerp_available():diff --cc tests/test_http_request.pyindex e14f8c8f4,0c10b27a0..233a5f0b2--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@@ -6,8 -6,14 +6,15 @@@ import warningfrom unittest import mockfrom urllib.parse import parse_qs, unquote_to_bytes, urlparse- from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse+ from scrapy.http import (+ Request,+ FormRequest,+ XmlRpcRequest,+ JsonRequest,+ Headers,+ HtmlResponse,+ )+from scrapy.http.request import NO_CALLBACKfrom scrapy.utils.python import to_bytes, to_unicode@@@ -278,25 -310,17 +311,25 @@@ class RequestTest(unittest.TestCase)self.assertIs(r4.callback, a_function)self.assertIs(r4.errback, a_function)+ r5 = self.request_class(- url='http://example.com',++ url="http://example.com",+ callback=NO_CALLBACK,+ )+ self.assertIs(r5.callback, NO_CALLBACK)+def test_callback_and_errback_type(self):with self.assertRaises(TypeError):- self.request_class('http://example.com', callback='a_function')+ self.request_class("http://example.com", callback="a_function")with self.assertRaises(TypeError):- self.request_class('http://example.com', errback='a_function')+ self.request_class("http://example.com", errback="a_function")with self.assertRaises(TypeError):self.request_class(- url='http://example.com',- callback='a_function',- errback='a_function',+ url="http://example.com",+ callback="a_function",+ errback="a_function",)+ with self.assertRaises(TypeError):- self.request_class('http://example.com', errback=NO_CALLBACK)++ self.request_class("http://example.com", errback=NO_CALLBACK)def test_from_curl(self):# Note: more curated tests regarding curl conversion are indiff --cc tests/test_pipeline_media.pyindex 99fb424f4,87ab03395..0a0c13a1c--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@@ -30,8 -31,7 +32,8 @@@ elsedef _mocked_download_func(request, info):+ assert request.callback is NO_CALLBACK- response = request.meta.get('response')+ response = request.meta.get("response")return response() if callable(response) else responsediff --cc tox.iniindex 1ff5f7a4e,eaf29173a..f2268b0f6--- a/tox.ini+++ b/tox.ini@@@ -4,7 -4,7 +4,7 @@@# and then run "tox" from this directory.[tox]- envlist = security,flake8,typing,py-envlist = security,flake8,py,black++envlist = security,flake8,black,typing,pyminversion = 1.7.0[testenv]@@@ -196,3 -195,10 +196,9 @@@ deps = {[docs]depssetenv = {[docs]setenv}commands =sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck++ [testenv:black]+ deps =+ black==22.12.0+ commands =+ black {posargs:--check .}-