Scrapy, a fast high-level web crawling & scraping framework for Python.
Add a genrequest command (#8058)
Adrian486e5a67dda4
diff --git a/docs/intro/install.rst b/docs/intro/install.rstindex 681076510..605c7fe87 100644--- a/docs/intro/install.rst+++ b/docs/intro/install.rst@@ -111,6 +111,8 @@ The following extras are available:- Provides* - ``bpython``- :ref:`bpython shell <shell-config>`+ * - ``clipboard``+ - Reading the system clipboard with :command:`genrequest`* - ``color``- :setting:`LOG_COLOR`* - ``gcs``diff --git a/docs/topics/commands.rst b/docs/topics/commands.rstindex 71a1d003d..d533fd367 100644--- a/docs/topics/commands.rst+++ b/docs/topics/commands.rst@@ -217,6 +217,7 @@ Global commands:* :command:`fetch`* :command:`view`* :command:`version`+* :command:`genrequest`* :command:`bench`Project-only commands:@@ -669,6 +670,27 @@ versionPrints the Scrapy version. If used with ``-v`` it also prints Python, Twistedand Platform info, which is useful for bug reports.+.. command:: genrequest++genrequest+----------++* Syntax: ``scrapy genrequest [curl command]``+* Requires project: *no*++.. versionadded:: VERSION++Print the Python code of a :class:`~scrapy.Request` object equivalent to a+`cURL <https://curl.se/>`_ command. If the curl command is not given as an+argument, it is read from the system clipboard, which requires the+``clipboard`` :ref:`extra <extras>`. If `ruff`_ is installed, it is used to+format the output.++Usage example::++ $ scrapy genrequest 'curl -d title=hello https://httpbin.org/post'+ Request(method="POST", url="https://httpbin.org/post", body="title=hello")+.. command:: benchbench@@ -805,6 +827,8 @@ For real examples, see the built-in Scrapy commands in the `scrapy/commands`_ di.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands+.. _ruff: https://docs.astral.sh/ruff/+.. autoclass:: scrapy.commands.ScrapyCommand:members::undoc-members:diff --git a/pyproject.toml b/pyproject.tomlindex a93fae1c3..cd7dbac7f 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -70,6 +70,7 @@ Tracker = "https://github.com/scrapy/scrapy/issues"[project.optional-dependencies]bpython = ["bpython>=0.7.1"]+clipboard = ["pyperclip>=1.8.0"]color = ["colorlog>=6.6.0"]gcs = ["google-cloud-storage>=1.30.0"]httpx = ["httpx2[http2,socks]>=2.0.0"]@@ -147,6 +148,7 @@ module = ["ipdb","pydispatch.*","pyftpdlib.*",+ "pyperclip","pytest_twisted","robotexclusionrulesparser","zope.interface.*",diff --git a/scrapy/commands/genrequest.py b/scrapy/commands/genrequest.pynew file mode 100644index 000000000..f0140bf4c--- /dev/null+++ b/scrapy/commands/genrequest.py@@ -0,0 +1,73 @@+from __future__ import annotations++import shutil+import subprocess+from typing import TYPE_CHECKING, Any, ClassVar++from scrapy.commands import ScrapyCommand+from scrapy.exceptions import UsageError+from scrapy.utils.curl import curl_to_request_kwargs++if TYPE_CHECKING:+ import argparse+++class Command(ScrapyCommand):+ requires_crawler_process = False+ default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}++ def syntax(self) -> str:+ return "[<curl command>]"++ def short_desc(self) -> str:+ return "Generate the Python code of a Request equivalent to a curl command"++ def long_desc(self) -> str:+ return (+ "Generate the Python code of a Request equivalent to a curl "+ "command. If the curl command is not given as an argument, it "+ "is read from the system clipboard."+ )++ def run(self, args: list[str], opts: argparse.Namespace) -> None:+ if len(args) > 1:+ raise UsageError+ curl_command = args[0] if args else _read_clipboard()+ try:+ request_kwargs = curl_to_request_kwargs(curl_command)+ except ValueError as e:+ raise UsageError(str(e), print_help=False) from e+ kwargs_repr = ", ".join(f"{k}={v!r}" for k, v in request_kwargs.items())+ code = f"Request({kwargs_repr})"+ print(_format_code(code).rstrip("\n"))+++def _format_code(code: str) -> str:+ ruff = shutil.which("ruff")+ if ruff is None:+ return code+ try:+ result = subprocess.run( # noqa: S603+ [ruff, "format", "-"],+ input=code,+ capture_output=True,+ text=True,+ check=True,+ )+ except subprocess.CalledProcessError:+ return code+ return result.stdout+++def _read_clipboard() -> str:+ try:+ import pyperclip # noqa: PLC0415+ except ImportError:+ raise UsageError(+ "No curl command given, and the pyperclip package is not "+ "installed to read one from the system clipboard. Pass the "+ "curl command as an argument, or install the scrapy[clipboard] "+ "extra.",+ print_help=False,+ ) from None+ return str(pyperclip.paste()).strip()diff --git a/tests/test_command_genrequest.py b/tests/test_command_genrequest.pynew file mode 100644index 000000000..b3c150b1a--- /dev/null+++ b/tests/test_command_genrequest.py@@ -0,0 +1,105 @@+from __future__ import annotations++import ast+import shutil+import subprocess+import sys+from types import SimpleNamespace+from typing import Any++import pytest++from scrapy.commands.genrequest import _format_code, _read_clipboard+from scrapy.exceptions import UsageError+from tests.utils.cmdline import proc+++def _parse_request_call(code: str) -> dict[str, Any]:+ """Return the keyword arguments of the single ``Request(...)`` call in *code*.++ Optional ruff formatting changes quote style and line wrapping, so tests+ compare the parsed call rather than the raw text.+ """+ call = ast.parse(code, mode="eval").body+ assert isinstance(call, ast.Call)+ return {+ kw.arg: ast.literal_eval(kw.value) for kw in call.keywords if kw.arg is not None+ }+++class TestGenRequestCommand:+ def test_get(self) -> None:+ _, out, _ = proc("genrequest", "curl http://example.com/")+ assert _parse_request_call(out) == {+ "method": "GET",+ "url": "http://example.com/",+ }++ def test_post_with_data(self) -> None:+ _, out, _ = proc("genrequest", "curl -d title=hello https://example.com/post")+ assert _parse_request_call(out) == {+ "method": "POST",+ "url": "https://example.com/post",+ "body": "title=hello",+ }++ def test_headers_and_cookies(self) -> None:+ _, out, _ = proc(+ "genrequest",+ "curl -H 'X-Test: 1' -b 'a=1' https://example.com/",+ )+ assert _parse_request_call(out) == {+ "method": "GET",+ "url": "https://example.com/",+ "headers": [("X-Test", "1")],+ "cookies": {"a": "1"},+ }++ def test_too_many_arguments(self) -> None:+ returncode, out, _ = proc("genrequest", "curl http://example.com/", "extra")+ assert returncode == 2+ assert "Usage" in out++ def test_invalid_curl_command(self) -> None:+ returncode, _, err = proc("genrequest", "not a curl command")+ assert returncode == 2+ assert "must start with" in err+++class TestReadClipboard:+ def test_pyperclip(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setitem(+ sys.modules, "pyperclip", SimpleNamespace(paste=lambda: "curl a.example ")+ )+ assert _read_clipboard() == "curl a.example"++ def test_pyperclip_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setitem(sys.modules, "pyperclip", None)+ with pytest.raises(UsageError, match="scrapy\\[clipboard\\]"):+ _read_clipboard()+++class TestFormatCode:+ CODE = "Request(method='GET', url='http://example.com/')"++ def test_ruff_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(shutil, "which", lambda name: None)+ assert _format_code(self.CODE) == self.CODE++ def test_ruff_formats(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/ruff")+ monkeypatch.setattr(+ subprocess,+ "run",+ lambda *a, **k: SimpleNamespace(stdout='Request(url="x")\n'),+ )+ assert _format_code(self.CODE) == 'Request(url="x")\n'++ def test_ruff_fails(self, monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/ruff")++ def _raise(*args: Any, **kwargs: Any) -> None:+ raise subprocess.CalledProcessError(1, "ruff")++ monkeypatch.setattr(subprocess, "run", _raise)+ assert _format_code(self.CODE) == self.CODEdiff --git a/tests/test_commands.py b/tests/test_commands.pyindex 3a8b05203..db07a3d4d 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -937,6 +937,7 @@ class TestHelpMessage(TestProjectBase):"runspider","version","genspider",+ "genrequest","check","bench",],