Scrapy, a fast high-level web crawling & scraping framework for Python.
fix(shell): Run IPython prompt in a thread under a running loop (#7816)
Laerte Pereira8caaac6ecbd4
diff --git a/pyproject.toml b/pyproject.tomlindex 576a42e5c..1cbd39946 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -69,8 +69,8 @@ brotli = [gcs = ["google-cloud-storage>=1.29.0"]httpx = ["httpx2[http2,socks]>=2.0.0"]images = ["Pillow>=8.3.2"]-ipython = ["ipython>=7.1.0"]-ptpython = ["ptpython>=2.0.1"]+ipython = ["ipython>=8.15.0"]+ptpython = ["ptpython>=3.0.23"]robotparser = ["robotexclusionrulesparser>=1.6.2"]s3 = ["boto3>=1.20.0"]twisted-http2 = ["Twisted[http2]>=21.7.0"]diff --git a/scrapy/utils/console.py b/scrapy/utils/console.pyindex 31a4bb32f..23b4401e8 100644--- a/scrapy/utils/console.py+++ b/scrapy/utils/console.py@@ -1,8 +1,9 @@from __future__ import annotations+import asyncioimport codefrom collections.abc import Callable-from functools import wraps+from functools import partial, wrapsfrom typing import TYPE_CHECKING, Anyif TYPE_CHECKING:@@ -16,16 +17,8 @@ def _embed_ipython_shell(namespace: dict[str, Any] | None = None, banner: str = "") -> EmbedFuncT:"""Start an IPython Shell"""- try:- from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415- from IPython.terminal.ipapp import load_default_config # noqa: PLC0415- except ImportError:- from IPython.frontend.terminal.embed import ( # type: ignore[import-not-found,no-redef] # noqa: T100,PLC0415- InteractiveShellEmbed,- )- from IPython.frontend.terminal.ipapp import ( # type: ignore[import-not-found,no-redef] # noqa: PLC0415- load_default_config,- )+ from IPython.terminal.embed import InteractiveShellEmbed # noqa: T100,PLC0415+ from IPython.terminal.ipapp import load_default_config # noqa: PLC0415@wraps(_embed_ipython_shell)def wrapper(namespace: dict[str, Any] = namespace or {}, banner: str = "") -> None:@@ -38,6 +31,19 @@ def _embed_ipython_shell(shell = InteractiveShellEmbed.instance(banner1=banner, user_ns=namespace, config=config)+ # If an asyncio event loop is already running in this thread, e.g. when+ # inspect_response() is called from a spider callback while using the+ # asyncio reactor, prompt_toolkit cannot run its own event loop here, so+ # ask it to run the prompt in a separate thread instead. pt_app is None+ # when IPython falls back to its simple prompt, which needs no event loop.+ # See https://github.com/scrapy/scrapy/issues/5447+ if (pt_app := getattr(shell, "pt_app", None)) is not None:+ try:+ asyncio.get_running_loop()+ except RuntimeError:+ pass+ else:+ pt_app.prompt = partial(pt_app.prompt, in_thread=True)shell()return wrapperdiff --git a/tests/test_utils_console.py b/tests/test_utils_console.pyindex ab0c72d8a..0dea8af6d 100644--- a/tests/test_utils_console.py+++ b/tests/test_utils_console.py@@ -1,10 +1,38 @@from __future__ import annotations+import subprocess+import sysfrom importlib.util import find_spec+from io import BytesIO+from typing import TYPE_CHECKINGimport pytest+from pexpect import EOFfrom scrapy.utils.console import get_shell_embed_func, start_python_console+from scrapy.utils.test import get_testenv++if TYPE_CHECKING:+ from pathlib import Path++CONSOLE = """+from scrapy.utils.console import start_python_console++start_python_console(banner="SHELL-READY", shells=["ipython"])+"""++CONSOLE_IN_RUNNING_LOOP = """+import asyncio++from scrapy.utils.console import start_python_console+++async def main():+ start_python_console(banner="SHELL-READY", shells=["ipython"])+++asyncio.run(main())+"""def test_get_shell_embed_func():@@ -61,6 +89,76 @@ def test_get_shell_embed_func_default():assert shell.__name__ == expected[email protected](find_spec("IPython") is None, reason="IPython is not installed")+class TestIPythonShell:+ """Starting an IPython shell, with and without an asyncio event loop already+ running in the calling thread. The latter happens when inspect_response() is+ called from a spider callback while using the asyncio reactor."""++ @staticmethod+ def _env(tmp_path: Path) -> dict[str, str]:+ env = get_testenv()+ # Keep IPython away from the profile and history of the user running the tests.+ env["IPYTHONDIR"] = str(tmp_path)+ return env++ def test_simple_prompt(self, tmp_path: Path) -> None:+ """IPython falls back to its simple prompt, which needs no event loop,+ when stdin is not a TTY."""+ env = self._env(tmp_path)+ p = subprocess.run(+ [sys.executable, "-c", CONSOLE_IN_RUNNING_LOOP],+ check=False,+ capture_output=True,+ encoding="utf-8",+ timeout=60,+ env=env,+ stdin=subprocess.DEVNULL,+ )+ output = p.stdout + p.stderr+ assert "SHELL-READY" in output+ assert p.returncode == 0, output++ @pytest.mark.skipif(+ sys.platform == "win32", reason="requires a POSIX pseudo-terminal"+ )+ @pytest.mark.parametrize(+ "script",+ [CONSOLE, CONSOLE_IN_RUNNING_LOOP],+ ids=["no_running_loop", "running_loop"],+ )+ def test_tty(self, tmp_path: Path, script: str) -> None:+ """IPython uses prompt_toolkit, which needs an event loop of its own,+ when stdin is a TTY."""+ # pexpect only defines spawn, which needs a pseudo-terminal, on POSIX.+ from pexpect import spawn # noqa: PLC0415++ env = self._env(tmp_path)+ env.pop("IPY_TEST_SIMPLE_PROMPT", None)+ env["TERM"] = "xterm"+ logfile = BytesIO()+ p = spawn(+ sys.executable,+ ["-c", script],+ env=env,+ timeout=60,+ )+ p.logfile_read = logfile+ try:+ # Wait for the prompt, which prompt_toolkit draws once it is done+ # querying the terminal, before typing into it.+ p.expect(r"In \[")+ p.sendline("21*2")+ p.expect_exact("42")+ p.sendline("exit()")+ p.expect(EOF)+ finally:+ p.close()+ output = logfile.getvalue().decode()+ assert "Traceback" not in output+ assert p.exitstatus == 0, output++def test_start_python_console_exit(monkeypatch: pytest.MonkeyPatch) -> None:def embed(namespace: dict[str, object], banner: str) -> None:raise SystemExitdiff --git a/tox.ini b/tox.iniindex b59221340..8331e2ab6 100644--- a/tox.ini+++ b/tox.ini@@ -190,8 +190,8 @@ deps =brotlicffi==1.2.0.0; implementation_name == "pypy"google-cloud-storage==1.29.0httpx2[http2,socks]==2.0.0- ipython==7.1.0- ptpython==2.0.1+ ipython==8.15.0+ ptpython==3.0.23robotexclusionrulesparser==1.6.2uvloop==0.16.0; platform_system != "Windows" and implementation_name != "pypy"zstandard==0.16.0; implementation_name != "pypy"