Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

fix(shell): Run IPython prompt in a thread under a running loop (#7816)

Laerte Pereira8caaac6ecbd4
diff --git a/pyproject.toml b/pyproject.toml
index 576a42e5c..1cbd39946 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -69,8 +69,8 @@ brotli = [
gcs = ["google-cloud-storage>=1.29.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
images = ["Pillow>=8.3.2"]
-ipython = ["ipython>=7.1.0"]
-ptpython = ["ptpython>=2.0.1"]
+ipython = ["ipython>=8.15.0"]
+ptpython = ["ptpython>=3.0.23"]
robotparser = ["robotexclusionrulesparser>=1.6.2"]
s3 = ["boto3>=1.20.0"]
twisted-http2 = ["Twisted[http2]>=21.7.0"]
diff --git a/scrapy/utils/console.py b/scrapy/utils/console.py
index 31a4bb32f..23b4401e8 100644
--- a/scrapy/utils/console.py
+++ b/scrapy/utils/console.py
@@ -1,8 +1,9 @@
from __future__ import annotations
+import asyncio
import code
from collections.abc import Callable
-from functools import wraps
+from functools import partial, wraps
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
@@ -16,16 +17,8 @@ def _embed_ipython_shell(
namespace: dict[str, Any] | None = None, banner: str = ""
) -> EmbedFuncT:
"""Start an IPython Shell"""
- try:
- from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
- from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
- except ImportError:
- from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415
- InteractiveShellEmbed,
- )
- from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415
- load_default_config,
- )
+ from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415
+ from IPython.terminal.ipapp import load_default_config # noqa: PLC0415
@wraps(_embed_ipython_shell)
def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:
@@ -38,6 +31,19 @@ def _embed_ipython_shell(
shell = InteractiveShellEmbed.instance(
banner1=banner, user_ns=namespace, config=config
)
+ # If an asyncio event loop is already running in this thread, e.g. when
+ # inspect_response() is called from a spider callback while using the
+ # asyncio reactor, prompt_toolkit cannot run its own event loop here, so
+ # ask it to run the prompt in a separate thread instead. pt_app is None
+ # when IPython falls back to its simple prompt, which needs no event loop.
+ # See https://github.com/scrapy/scrapy/issues/5447
+ if (pt_app := getattr(shell, "pt_app", None)) is not None:
+ try:
+ asyncio.get_running_loop()
+ except RuntimeError:
+ pass
+ else:
+ pt_app.prompt = partial(pt_app.prompt, in_thread=True)
shell()
return wrapper
diff --git a/tests/test_utils_console.py b/tests/test_utils_console.py
index ab0c72d8a..0dea8af6d 100644
--- a/tests/test_utils_console.py
+++ b/tests/test_utils_console.py
@@ -1,10 +1,38 @@
from __future__ import annotations
+import subprocess
+import sys
from importlib.util import find_spec
+from io import BytesIO
+from typing import TYPE_CHECKING
import pytest
+from pexpect import EOF
from scrapy.utils.console import get_shell_embed_func, start_python_console
+from scrapy.utils.test import get_testenv
+
+if TYPE_CHECKING:
+ from pathlib import Path
+
+CONSOLE = """
+from scrapy.utils.console import start_python_console
+
+start_python_console(banner="SHELL-READY", shells=["ipython"])
+"""
+
+CONSOLE_IN_RUNNING_LOOP = """
+import asyncio
+
+from scrapy.utils.console import start_python_console
+
+
+async def main():
+ start_python_console(banner="SHELL-READY", shells=["ipython"])
+
+
+asyncio.run(main())
+"""
def test_get_shell_embed_func():
@@ -61,6 +89,76 @@ def test_get_shell_embed_func_default():
assert shell.__name__ == expected
[email protected](find_spec("IPython") is None, reason="IPython is not installed")
+class TestIPythonShell:
+ """Starting an IPython shell, with and without an asyncio event loop already
+ running in the calling thread. The latter happens when inspect_response() is
+ called from a spider callback while using the asyncio reactor."""
+
+ @staticmethod
+ def _env(tmp_path: Path) -> dict[str, str]:
+ env = get_testenv()
+ # Keep IPython away from the profile and history of the user running the tests.
+ env["IPYTHONDIR"] = str(tmp_path)
+ return env
+
+ def test_simple_prompt(self, tmp_path: Path) -> None:
+ """IPython falls back to its simple prompt, which needs no event loop,
+ when stdin is not a TTY."""
+ env = self._env(tmp_path)
+ p = subprocess.run(
+ [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],
+ check=False,
+ capture_output=True,
+ encoding="utf-8",
+ timeout=60,
+ env=env,
+ stdin=subprocess.DEVNULL,
+ )
+ output = p.stdout + p.stderr
+ assert "SHELL-READY" in output
+ assert p.returncode == 0, output
+
+ @pytest.mark.skipif(
+ sys.platform == "win32", reason="requires a POSIX pseudo-terminal"
+ )
+ @pytest.mark.parametrize(
+ "script",
+ [CONSOLE, CONSOLE_IN_RUNNING_LOOP],
+ ids=["no_running_loop", "running_loop"],
+ )
+ def test_tty(self, tmp_path: Path, script: str) -> None:
+ """IPython uses prompt_toolkit, which needs an event loop of its own,
+ when stdin is a TTY."""
+ # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.
+ from pexpect import spawn # noqa: PLC0415
+
+ env = self._env(tmp_path)
+ env.pop("IPY_TEST_SIMPLE_PROMPT", None)
+ env["TERM"] = "xterm"
+ logfile = BytesIO()
+ p = spawn(
+ sys.executable,
+ ["-c", script],
+ env=env,
+ timeout=60,
+ )
+ p.logfile_read = logfile
+ try:
+ # Wait for the prompt, which prompt_toolkit draws once it is done
+ # querying the terminal, before typing into it.
+ p.expect(r"In \[")
+ p.sendline("21*2")
+ p.expect_exact("42")
+ p.sendline("exit()")
+ p.expect(EOF)
+ finally:
+ p.close()
+ output = logfile.getvalue().decode()
+ assert "Traceback" not in output
+ assert p.exitstatus == 0, output
+
+
def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:
def embed(namespace: dict[str, object], banner: str) -> None:
raise SystemExit
diff --git a/tox.ini b/tox.ini
index b59221340..8331e2ab6 100644
--- a/tox.ini
+++ b/tox.ini
@@ -190,8 +190,8 @@ deps =
brotlicffi==1.2.0.0; implementation_name == "pypy"
google-cloud-storage==1.29.0
httpx2[http2,socks]==2.0.0
- ipython==7.1.0
- ptpython==2.0.1
+ ipython==8.15.0
+ ptpython==3.0.23
robotexclusionrulesparser==1.6.2
uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"
zstandard==0.16.0; implementation_name != "pypy"