Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add an interactive test for `scrapy shell`.

Andrey Rakhmatullind19e315b0b07
diff --git a/tests/requirements.txt b/tests/requirements.txt
index 3ea7f3333..dc004f3f1 100644
--- a/tests/requirements.txt
+++ b/tests/requirements.txt
@@ -1,5 +1,6 @@
# Tests requirements
attrs
+pexpect >= 4.8.0
# https://github.com/giampaolo/pyftpdlib/issues/560
pyftpdlib; python_version < "3.12"
pytest
diff --git a/tests/test_command_shell.py b/tests/test_command_shell.py
index 6589381f3..7d87eb62c 100644
--- a/tests/test_command_shell.py
+++ b/tests/test_command_shell.py
@@ -1,11 +1,15 @@
+import sys
+from io import BytesIO
from pathlib import Path
+from pexpect.popen_spawn import PopenSpawn
from twisted.internet import defer
from twisted.trial import unittest
from scrapy.utils.testproc import ProcessTest
from scrapy.utils.testsite import SiteTest
from tests import NON_EXISTING_RESOLVABLE, tests_datadir
+from tests.mockserver import MockServer
class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
@@ -133,3 +137,25 @@ class ShellTest(ProcessTest, SiteTest, unittest.TestCase):
args = ["-c", code, "--set", f"TWISTED_REACTOR={reactor_path}"]
_, _, err = yield self.execute(args, check_code=True)
self.assertNotIn(b"RuntimeError: There is no current event loop in thread", err)
+
+
+class InteractiveShellTest(unittest.TestCase):
+ def test_fetch(self):
+ args = (
+ sys.executable,
+ "-m",
+ "scrapy.cmdline",
+ "shell",
+ )
+ logfile = BytesIO()
+ p = PopenSpawn(args, timeout=5)
+ p.logfile_read = logfile
+ p.expect_exact("Available Scrapy objects")
+ with MockServer() as mockserver:
+ p.sendline(f"fetch('{mockserver.url('/')}')")
+ p.sendline("type(response)")
+ p.expect_exact("HtmlResponse")
+ p.sendeof()
+ p.wait()
+ logfile.seek(0)
+ self.assertNotIn("Traceback", logfile.read().decode())