Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Switch the shell to download_async() (#8176)

Andrey Rakhmatullin3a799fd70127
diff --git a/docs/news.rst b/docs/news.rst
index dcfc164e0..3581c4240 100644
--- a/docs/news.rst
+++ b/docs/news.rst
@@ -3,6 +3,27 @@
Release notes
=============
+Scrapy VERSION (unreleased)
+---------------------------
+
+Backward-incompatible changes
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+- Changes to the ``fetch()`` shortcut of the :ref:`Scrapy shell
+ <topics-shell>`:
+
+ - Requests are now downloaded with
+ :meth:`ExecutionEngine.download_async()
+ <scrapy.core.engine.ExecutionEngine.download_async>` instead of being
+ passed to the scheduler. As a result, spider middlewares are no longer
+ applied to them.
+
+ - When a :class:`~scrapy.Request` object is passed, its ``callback`` and
+ ``errback`` are no longer called, and a warning is issued if they are
+ set.
+
+ (:gh:`TBD`)
+
.. _release-2.19.0:
Scrapy 2.19.0 (2026-09-10)
diff --git a/scrapy/commands/shell.py b/scrapy/commands/shell.py
index 35a3ed423..3574d8e62 100644
--- a/scrapy/commands/shell.py
+++ b/scrapy/commands/shell.py
@@ -29,6 +29,7 @@ class Command(ScrapyCommand):
default_settings: ClassVar[dict[str, Any]] = {
"DUPEFILTER_CLASS": "scrapy.dupefilters.BaseDupeFilter",
"LOGSTATS_INTERVAL": 0,
+ "TELNETCONSOLE_ENABLED": False,
"REMOTE_CONTROL_ENABLED": False,
}
diff --git a/scrapy/shell.py b/scrapy/shell.py
index 294b43893..f22c80167 100644
--- a/scrapy/shell.py
+++ b/scrapy/shell.py
@@ -15,7 +15,6 @@ from typing import TYPE_CHECKING, Any
from itemadapter import is_item
from twisted.internet import threads
-from twisted.internet.defer import Deferred
from twisted.python import threadable
from w3lib.url import any_to_uri
@@ -27,8 +26,7 @@ from scrapy.settings import Settings
from scrapy.spiders import Spider
from scrapy.utils._shell import DEFAULT_PYTHON_SHELLS, start_python_console
from scrapy.utils.conf import get_config
-from scrapy.utils.datatypes import SequenceExclude
-from scrapy.utils.defer import deferred_f_from_coro_f, maybe_deferred_to_future
+from scrapy.utils.defer import deferred_f_from_coro_f
from scrapy.utils.misc import load_object
from scrapy.utils.response import open_in_browser
@@ -66,18 +64,13 @@ if TYPE_CHECKING:
# 4. Shell._schedule() calls Shell._open_spider() (on the first call).
# 5. Shell._open_spider() calls
# engine.open_spider_async(close_if_idle=False).
-# 6. Shell._schedule() calls engine.crawl(request), scheduling the request.
-# 7. Shell._schedule() via _request_deferred() waits until the request callback
-# is called. When it's called, the response becomes available.
+# 6. Shell._schedule() calls engine.download_async(request), which completes
+# with the response.
#
# In the reactorless mode this is slightly different, the engine initialization
# happens in the event loop thread as many things need either a reactor or a
# running event loop.
#
-# Side note: it should be possible to remove _request_deferred() by using
-# engine.download_async() instead of engine.crawl(), losing the usual stuff
-# like spider middlewares (none of which should be important).
-#
# Other architecture problems:
# * scrapy.cmdline.execute() creates an AsyncCrawlerProcess instance which
# immediately installs a reactor (which is maybe not thread-specific?) or an
@@ -193,10 +186,7 @@ class Shell:
"""
if not self.spider:
await self._open_spider(spider)
- # send the request to the engine
- self.crawler.engine.crawl(request)
- # this will fire when the request callback runs (via the callback hijacking in _request_deferred())
- return await maybe_deferred_to_future(_request_deferred(request))
+ return await self.crawler.engine.download_async(request)
async def _open_spider(self, spider: Spider | None) -> None:
if spider is None:
@@ -215,15 +205,16 @@ class Shell:
) -> None:
if isinstance(request_or_url, Request):
request = request_or_url
+ if request.callback or request.errback:
+ warnings.warn(
+ "Callbacks and errbacks of Request objects passed to fetch() are ignored.",
+ stacklevel=2,
+ )
else:
url = any_to_uri(request_or_url)
request = Request(url, dont_filter=True, **kwargs)
- if redirect:
- request.meta["handle_httpstatus_list"] = SequenceExclude(
- range(300, 400)
- )
- else:
- request.meta["handle_httpstatus_all"] = True
+ if not redirect:
+ request.meta["dont_redirect"] = True
response: Response | None = None
if self._use_reactor:
from twisted.internet import reactor
@@ -307,33 +298,3 @@ def inspect_response(response: Response, spider: Spider) -> None:
loop = None
Shell(spider.crawler, loop=loop).start(response=response, spider=spider)
signal.signal(signal.SIGINT, sigint_handler)
-
-
-def _request_deferred(request: Request) -> Deferred[Any]:
- """Wrap a request inside a Deferred.
-
- This function is harmful, do not use it until you know what you are doing.
-
- This returns a Deferred whose first pair of callbacks are the request
- callback and errback. The Deferred also triggers when the request
- callback/errback is executed (i.e. when the request is downloaded)
-
- WARNING: Do not call request.replace() until after the deferred is called.
- """
- request_callback = request.callback
- request_errback = request.errback
-
- def _restore_callbacks(result: Any) -> Any:
- request.callback = request_callback
- request.errback = request_errback
- return result
-
- d: Deferred[Any] = Deferred()
- d.addBoth(_restore_callbacks)
- if request.callback:
- d.addCallback(request.callback)
- if request.errback:
- d.addErrback(request.errback)
-
- request.callback, request.errback = d.callback, d.errback
- return d
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 979704a87..b101e732b 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -157,10 +157,24 @@ class TestShellCommand:
def test_fetch_request_with_callbacks(self, mockserver: MockServer) -> None:
url = mockserver.url("/text")
code = (
- f"fetch(scrapy.Request('{url}', callback=lambda r: r, errback=lambda f: f))"
+ f"fetch(scrapy.Request('{url}', callback=lambda r: print('CALLBACK'), "
+ "errback=lambda f: print('ERRBACK')))"
)
- ret, out, _ = proc("shell", "-c", code)
+ ret, out, err = proc("shell", "-c", code)
assert ret == 0, out
+ assert "CALLBACK" not in out
+ assert "ERRBACK" not in out
+ assert (
+ "UserWarning: Callbacks and errbacks of Request objects passed to fetch() are ignored"
+ in err
+ )
+
+ def test_redirect_referer(self, mockserver: MockServer) -> None:
+ """Redirects set the Referer header even in the shell."""
+ url = mockserver.url("/redirect-no-meta-refresh")
+ code = f"fetch('{url}') or response.request.headers.get('Referer')"
+ _, out, _ = proc("shell", "-c", code)
+ assert url in out
class TestShellCommandWithSpider(TestProjectBase):