Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge remote-tracking branch 'scrapy/master' into no-callback

Adrián Chaves9272c4af0c60
diff --cc scrapy/downloadermiddlewares/robotstxt.py
index 67e14b7b5,326c35290..23b5924e1
--- a/scrapy/downloadermiddlewares/robotstxt.py
+++ b/scrapy/downloadermiddlewares/robotstxt.py
@@@ -66,8 -68,7 +69,8 @@@ class RobotsTxtMiddleware
robotsreq = Request(
robotsurl,
priority=self.DOWNLOAD_PRIORITY,
- meta={'dont_obey_robotstxt': True},
+ meta={"dont_obey_robotstxt": True},
+ callback=NO_CALLBACK,
)
dfd = self.crawler.engine.download(robotsreq)
dfd.addCallback(self._parse_robots, netloc, spider)
diff --cc scrapy/http/request/__init__.py
index 936afb007,e290f2143..a78ba5115
--- a/scrapy/http/request/__init__.py
+++ b/scrapy/http/request/__init__.py
@@@ -95,15 -90,6 +105,15 @@@ class Request(object_ref)
self._cb_kwargs = dict(cb_kwargs) if cb_kwargs else None
self.flags = [] if flags is None else list(flags)
+ def _set_xback(self, name: str, value: Optional[Callable]) -> None:
+ if (
+ value is not None
+ and (name != "callback" or value is not NO_CALLBACK)
+ and not callable(value)
+ ):
- raise TypeError(f'{name} must be a callable, got {type(value).__name__}')
++ raise TypeError(f"{name} must be a callable, got {type(value).__name__}")
+ setattr(self, name, value)
+
@property
def cb_kwargs(self) -> dict:
if self._cb_kwargs is None:
diff --cc setup.py
index 049c4f9a6,f53334d4e..9150dac0b
--- a/setup.py
+++ b/setup.py
@@@ -18,23 -18,22 +18,23 @@@ def has_environment_marker_platform_imp
install_requires = [
- 'Twisted>=18.9.0',
- 'cryptography>=3.4.6',
- 'cssselect>=0.9.1',
- 'itemloaders>=1.0.1',
- 'parsel>=1.5.0',
- 'pyOpenSSL>=21.0.0',
- 'queuelib>=1.4.2',
- 'service_identity>=18.1.0',
- 'w3lib>=1.17.0',
- 'zope.interface>=5.1.0',
- 'protego>=0.1.15',
- 'itemadapter>=0.1.0',
- 'setuptools',
- 'packaging',
- 'tldextract',
- 'lxml>=4.3.0',
- 'typing-extensions>=3.10.0.0',
+ "Twisted>=18.9.0",
+ "cryptography>=3.4.6",
+ "cssselect>=0.9.1",
+ "itemloaders>=1.0.1",
+ "parsel>=1.5.0",
+ "pyOpenSSL>=21.0.0",
+ "queuelib>=1.4.2",
+ "service_identity>=18.1.0",
+ "w3lib>=1.17.0",
+ "zope.interface>=5.1.0",
+ "protego>=0.1.15",
+ "itemadapter>=0.1.0",
+ "setuptools",
+ "packaging",
+ "tldextract",
+ "lxml>=4.3.0",
++ "typing-extensions>=3.10.0.0",
]
extras_require = {}
cpython_dependencies = [
diff --cc tests/test_downloadermiddleware_robotstxt.py
index 71d53ff1a,ac08c6006..953b61626
--- a/tests/test_downloadermiddleware_robotstxt.py
+++ b/tests/test_downloadermiddleware_robotstxt.py
@@@ -4,11 -4,12 +4,13 @@@ from twisted.internet import reactor, e
from twisted.internet.defer import Deferred, DeferredList, maybeDeferred
from twisted.python import failure
from twisted.trial import unittest
- from scrapy.downloadermiddlewares.robotstxt import (RobotsTxtMiddleware,
- logger as mw_module_logger)
+ from scrapy.downloadermiddlewares.robotstxt import (
+ RobotsTxtMiddleware,
+ logger as mw_module_logger,
+ )
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request, Response, TextResponse
+from scrapy.http.request import NO_CALLBACK
from scrapy.settings import Settings
from tests.test_robotstxt_interface import rerp_available, reppy_available
@@@ -52,14 -55,21 +56,22 @@@ Disallow: /some/randome/page.htm
def test_robotstxt(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
- return DeferredList([
- self.assertNotIgnored(Request('http://site.local/allowed'), middleware),
- maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),
- self.assertIgnored(Request('http://site.local/admin/main'), middleware),
- self.assertIgnored(Request('http://site.local/static/'), middleware),
- self.assertIgnored(Request('http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:'), middleware),
- self.assertIgnored(Request('http://site.local/wiki/Käyttäjä:'), middleware)
- ], fireOnOneErrback=True)
+ return DeferredList(
+ [
+ self.assertNotIgnored(Request("http://site.local/allowed"), middleware),
++ maybeDeferred(self.assertRobotsTxtRequested, "http://site.local"),
+ self.assertIgnored(Request("http://site.local/admin/main"), middleware),
+ self.assertIgnored(Request("http://site.local/static/"), middleware),
+ self.assertIgnored(
+ Request("http://site.local/wiki/K%C3%A4ytt%C3%A4j%C3%A4:"),
+ middleware,
+ ),
+ self.assertIgnored(
+ Request("http://site.local/wiki/Käyttäjä:"), middleware
+ ),
+ ],
+ fireOnOneErrback=True,
+ )
def test_robotstxt_ready_parser(self):
middleware = RobotsTxtMiddleware(self._get_successful_crawler())
@@@ -182,15 -222,10 +224,16 @@@
def assertIgnored(self, request, middleware):
spider = None # not actually used
- return self.assertFailure(maybeDeferred(middleware.process_request, request, spider),
- IgnoreRequest)
+ return self.assertFailure(
+ maybeDeferred(middleware.process_request, request, spider), IgnoreRequest
+ )
+ def assertRobotsTxtRequested(self, base_url):
+ calls = self.crawler.engine.download.call_args_list
+ request = calls[0][0][0]
+ self.assertEqual(request.url, f"{base_url}/robots.txt")
+ self.assertEqual(request.callback, NO_CALLBACK)
+
class RobotsTxtMiddlewareWithRerpTest(RobotsTxtMiddlewareTest):
if not rerp_available():
diff --cc tests/test_http_request.py
index e14f8c8f4,0c10b27a0..233a5f0b2
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@@ -6,8 -6,14 +6,15 @@@ import warning
from unittest import mock
from urllib.parse import parse_qs, unquote_to_bytes, urlparse
- from scrapy.http import Request, FormRequest, XmlRpcRequest, JsonRequest, Headers, HtmlResponse
+ from scrapy.http import (
+ Request,
+ FormRequest,
+ XmlRpcRequest,
+ JsonRequest,
+ Headers,
+ HtmlResponse,
+ )
+from scrapy.http.request import NO_CALLBACK
from scrapy.utils.python import to_bytes, to_unicode
@@@ -278,25 -310,17 +311,25 @@@ class RequestTest(unittest.TestCase)
self.assertIs(r4.callback, a_function)
self.assertIs(r4.errback, a_function)
+ r5 = self.request_class(
- url='http://example.com',
++ url="http://example.com",
+ callback=NO_CALLBACK,
+ )
+ self.assertIs(r5.callback, NO_CALLBACK)
+
def test_callback_and_errback_type(self):
with self.assertRaises(TypeError):
- self.request_class('http://example.com', callback='a_function')
+ self.request_class("http://example.com", callback="a_function")
with self.assertRaises(TypeError):
- self.request_class('http://example.com', errback='a_function')
+ self.request_class("http://example.com", errback="a_function")
with self.assertRaises(TypeError):
self.request_class(
- url='http://example.com',
- callback='a_function',
- errback='a_function',
+ url="http://example.com",
+ callback="a_function",
+ errback="a_function",
)
+ with self.assertRaises(TypeError):
- self.request_class('http://example.com', errback=NO_CALLBACK)
++ self.request_class("http://example.com", errback=NO_CALLBACK)
def test_from_curl(self):
# Note: more curated tests regarding curl conversion are in
diff --cc tests/test_pipeline_media.py
index 99fb424f4,87ab03395..0a0c13a1c
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@@ -30,8 -31,7 +32,8 @@@ else
def _mocked_download_func(request, info):
+ assert request.callback is NO_CALLBACK
- response = request.meta.get('response')
+ response = request.meta.get("response")
return response() if callable(response) else response
diff --cc tox.ini
index 1ff5f7a4e,eaf29173a..f2268b0f6
--- a/tox.ini
+++ b/tox.ini
@@@ -4,7 -4,7 +4,7 @@@
# and then run "tox" from this directory.
[tox]
- envlist = security,flake8,typing,py
-envlist = security,flake8,py,black
++envlist = security,flake8,black,typing,py
minversion = 1.7.0
[testenv]
@@@ -196,3 -195,10 +196,9 @@@ deps = {[docs]deps
setenv = {[docs]setenv}
commands =
sphinx-build -W -b linkcheck . {envtmpdir}/linkcheck
+
+ [testenv:black]
+ deps =
+ black==22.12.0
+ commands =
+ black {posargs:--check .}
-