Scrapy, a fast high-level web crawling & scraping framework for Python.
Switch the shell to download_async() (#8176)
Andrey Rakhmatullin3a799fd70127
diff --git a/docs/news.rst b/docs/news.rstindex dcfc164e0..3581c4240 100644--- a/docs/news.rst+++ b/docs/news.rst@@ -3,6 +3,27 @@Release notes=============+Scrapy VERSION (unreleased)+---------------------------++Backward-incompatible changes+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++- Changes to the ``fetch()`` shortcut of the :ref:`Scrapy shell+ <topics-shell>`:++ - Requests are now downloaded with+ :meth:`ExecutionEngine.download_async()+ <scrapy.core.engine.ExecutionEngine.download_async>` instead of being+ passed to the scheduler. As a result, spider middlewares are no longer+ applied to them.++ - When a :class:`~scrapy.Request` object is passed, its ``callback`` and+ ``errback`` are no longer called, and a warning is issued if they are+ set.++ (:gh:`TBD`)+.. _release-2.19.0:Scrapy 2.19.0 (2026-09-10)diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.pyindex 35a3ed423..3574d8e62 100644--- a/scrapy/commands/shell.py+++ b/scrapy/commands/shell.py@@ -29,6 +29,7 @@ class Command(ScrapyCommand):default_settings: ClassVar[dict[str, Any]] = {"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter","LOGSTATS_INTERVAL": 0,+ "TELNETCONSOLE_ENABLED": False,"REMOTE_CONTROL_ENABLED": False,}diff --git a/scrapy/shell.py b/scrapy/shell.pyindex 294b43893..f22c80167 100644--- a/scrapy/shell.py+++ b/scrapy/shell.py@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Anyfrom itemadapter import is_itemfrom twisted.internet import threads-from twisted.internet.defer import Deferredfrom twisted.python import threadablefrom w3lib.url import any_to_uri@@ -27,8 +26,7 @@ from scrapy.settings import Settingsfrom scrapy.spiders import Spiderfrom scrapy.utils._shell import DEFAULT_PYTHON_SHELLS, start_python_consolefrom scrapy.utils.conf import get_config-from scrapy.utils.datatypes import SequenceExclude-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future+from scrapy.utils.defer import deferred_f_from_coro_ffrom scrapy.utils.misc import load_objectfrom scrapy.utils.response import open_in_browser@@ -66,18 +64,13 @@ if TYPE_CHECKING:# 4. Shell._schedule() calls Shell._open_spider() (on the first call).# 5. Shell._open_spider() calls# engine.open_spider_async(close_if_idle=False).-# 6. Shell._schedule() calls engine.crawl(request), scheduling the request.-# 7. Shell._schedule() via _request_deferred() waits until the request callback-# is called. When it's called, the response becomes available.+# 6. Shell._schedule() calls engine.download_async(request), which completes+# with the response.## In the reactorless mode this is slightly different, the engine initialization# happens in the event loop thread as many things need either a reactor or a# running event loop.#-# Side note: it should be possible to remove _request_deferred() by using-# engine.download_async() instead of engine.crawl(), losing the usual stuff-# like spider middlewares (none of which should be important).-## Other architecture problems:# * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which# immediately installs a reactor (which is maybe not thread-specific?) or an@@ -193,10 +186,7 @@ class Shell:"""if not self.spider:await self._open_spider(spider)- # send the request to the engine- self.crawler.engine.crawl(request)- # this will fire when the request callback runs (via the callback hijacking in _request_deferred())- return await maybe_deferred_to_future(_request_deferred(request))+ return await self.crawler.engine.download_async(request)async def _open_spider(self, spider: Spider | None) -> None:if spider is None:@@ -215,15 +205,16 @@ class Shell:) -> None:if isinstance(request_or_url, Request):request = request_or_url+ if request.callback or request.errback:+ warnings.warn(+ "Callbacks and errbacks of Request objects passed to fetch() are ignored.",+ stacklevel=2,+ )else:url = any_to_uri(request_or_url)request = Request(url, dont_filter=True, **kwargs)- if redirect:- request.meta["handle_httpstatus_list"] = SequenceExclude(- range(300, 400)- )- else:- request.meta["handle_httpstatus_all"] = True+ if not redirect:+ request.meta["dont_redirect"] = Trueresponse: Response | None = Noneif self._use_reactor:from twisted.internet import reactor@@ -307,33 +298,3 @@ def inspect_response(response: Response, spider: Spider) -> None:loop = NoneShell(spider.crawler, loop=loop).start(response=response, spider=spider)signal.signal(signal.SIGINT, sigint_handler)---def _request_deferred(request: Request) -> Deferred[Any]:- """Wrap a request inside a Deferred.-- This function is harmful, do not use it until you know what you are doing.-- This returns a Deferred whose first pair of callbacks are the request- callback and errback. The Deferred also triggers when the request- callback/errback is executed (i.e. when the request is downloaded)-- WARNING: Do not call request.replace() until after the deferred is called.- """- request_callback = request.callback- request_errback = request.errback-- def _restore_callbacks(result: Any) -> Any:- request.callback = request_callback- request.errback = request_errback- return result-- d: Deferred[Any] = Deferred()- d.addBoth(_restore_callbacks)- if request.callback:- d.addCallback(request.callback)- if request.errback:- d.addErrback(request.errback)-- request.callback, request.errback = d.callback, d.errback- return ddiff --git a/tests/test_command_shell.py b/tests/test_command_shell.pyindex 979704a87..b101e732b 100644--- a/tests/test_command_shell.py+++ b/tests/test_command_shell.py@@ -157,10 +157,24 @@ class TestShellCommand:def test_fetch_request_with_callbacks(self, mockserver: MockServer) -> None:url = mockserver.url("/text")code = (- f"fetch(scrapy.Request('{url}', callback=lambda r: r, errback=lambda f: f))"+ f"fetch(scrapy.Request('{url}', callback=lambda r: print('CALLBACK'), "+ "errback=lambda f: print('ERRBACK')))")- ret, out, _ = proc("shell", "-c", code)+ ret, out, err = proc("shell", "-c", code)assert ret == 0, out+ assert "CALLBACK" not in out+ assert "ERRBACK" not in out+ assert (+ "UserWarning: Callbacks and errbacks of Request objects passed to fetch() are ignored"+ in err+ )++ def test_redirect_referer(self, mockserver: MockServer) -> None:+ """Redirects set the Referer header even in the shell."""+ url = mockserver.url("/redirect-no-meta-refresh")+ code = f"fetch('{url}') or response.request.headers.get('Referer')"+ _, out, _ = proc("shell", "-c", code)+ assert url in outclass TestShellCommandWithSpider(TestProjectBase):