Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add a genrequest command (#8058)

Adrian486e5a67dda4
diff --git a/docs/intro/install.rst b/docs/intro/install.rst
index 681076510..605c7fe87 100644
--- a/docs/intro/install.rst
+++ b/docs/intro/install.rst
@@ -111,6 +111,8 @@ The following extras are available:
- Provides
* - ``bpython``
- :ref:`bpython shell <shell-config>`
+ * - ``clipboard``
+ - Reading the system clipboard with :command:`genrequest`
* - ``color``
- :setting:`LOG_COLOR`
* - ``gcs``
diff --git a/docs/topics/commands.rst b/docs/topics/commands.rst
index 71a1d003d..d533fd367 100644
--- a/docs/topics/commands.rst
+++ b/docs/topics/commands.rst
@@ -217,6 +217,7 @@ Global commands:
* :command:`fetch`
* :command:`view`
* :command:`version`
+* :command:`genrequest`
* :command:`bench`
Project-only commands:
@@ -669,6 +670,27 @@ version
Prints the Scrapy version. If used with ``-v`` it also prints Python, Twisted
and Platform info, which is useful for bug reports.
+.. command:: genrequest
+
+genrequest
+----------
+
+* Syntax: ``scrapy genrequest [curl command]``
+* Requires project: *no*
+
+.. versionadded:: VERSION
+
+Print the Python code of a :class:`~scrapy.Request` object equivalent to a
+`cURL <https://curl.se/>`_ command. If the curl command is not given as an
+argument, it is read from the system clipboard, which requires the
+``clipboard`` :ref:`extra <extras>`. If `ruff`_ is installed, it is used to
+format the output.
+
+Usage example::
+
+ $ scrapy genrequest 'curl -d title=hello https://httpbin.org/post'
+ Request(method="POST", url="https://httpbin.org/post", body="title=hello")
+
.. command:: bench
bench
@@ -805,6 +827,8 @@ For real examples, see the built-in Scrapy commands in the `scrapy/commands`_ di
.. _scrapy/commands: https://github.com/scrapy/scrapy/tree/master/scrapy/commands
+.. _ruff: https://docs.astral.sh/ruff/
+
.. autoclass:: scrapy.commands.ScrapyCommand
:members:
:undoc-members:
diff --git a/pyproject.toml b/pyproject.toml
index a93fae1c3..cd7dbac7f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -70,6 +70,7 @@ Tracker = "https://github.com/scrapy/scrapy/issues"
[project.optional-dependencies]
bpython = ["bpython>=0.7.1"]
+clipboard = ["pyperclip>=1.8.0"]
color = ["colorlog>=6.6.0"]
gcs = ["google-cloud-storage>=1.30.0"]
httpx = ["httpx2[http2,socks]>=2.0.0"]
@@ -147,6 +148,7 @@ module = [
"ipdb",
"pydispatch.*",
"pyftpdlib.*",
+ "pyperclip",
"pytest_twisted",
"robotexclusionrulesparser",
"zope.interface.*",
diff --git a/scrapy/commands/genrequest.py b/scrapy/commands/genrequest.py
new file mode 100644
index 000000000..f0140bf4c
--- /dev/null
+++ b/scrapy/commands/genrequest.py
@@ -0,0 +1,73 @@
+from __future__ import annotations
+
+import shutil
+import subprocess
+from typing import TYPE_CHECKING, Any, ClassVar
+
+from scrapy.commands import ScrapyCommand
+from scrapy.exceptions import UsageError
+from scrapy.utils.curl import curl_to_request_kwargs
+
+if TYPE_CHECKING:
+ import argparse
+
+
+class Command(ScrapyCommand):
+ requires_crawler_process = False
+ default_settings: ClassVar[dict[str, Any]] = {"LOG_ENABLED": False}
+
+ def syntax(self) -> str:
+ return "[<curl command>]"
+
+ def short_desc(self) -> str:
+ return "Generate the Python code of a Request equivalent to a curl command"
+
+ def long_desc(self) -> str:
+ return (
+ "Generate the Python code of a Request equivalent to a curl "
+ "command. If the curl command is not given as an argument, it "
+ "is read from the system clipboard."
+ )
+
+ def run(self, args: list[str], opts: argparse.Namespace) -> None:
+ if len(args) > 1:
+ raise UsageError
+ curl_command = args[0] if args else _read_clipboard()
+ try:
+ request_kwargs = curl_to_request_kwargs(curl_command)
+ except ValueError as e:
+ raise UsageError(str(e), print_help=False) from e
+ kwargs_repr = ", ".join(f"{k}={v!r}" for k, v in request_kwargs.items())
+ code = f"Request({kwargs_repr})"
+ print(_format_code(code).rstrip("\n"))
+
+
+def _format_code(code: str) -> str:
+ ruff = shutil.which("ruff")
+ if ruff is None:
+ return code
+ try:
+ result = subprocess.run( # noqa: S603
+ [ruff, "format", "-"],
+ input=code,
+ capture_output=True,
+ text=True,
+ check=True,
+ )
+ except subprocess.CalledProcessError:
+ return code
+ return result.stdout
+
+
+def _read_clipboard() -> str:
+ try:
+ import pyperclip # noqa: PLC0415
+ except ImportError:
+ raise UsageError(
+ "No curl command given, and the pyperclip package is not "
+ "installed to read one from the system clipboard. Pass the "
+ "curl command as an argument, or install the scrapy[clipboard] "
+ "extra.",
+ print_help=False,
+ ) from None
+ return str(pyperclip.paste()).strip()
diff --git a/tests/test_command_genrequest.py b/tests/test_command_genrequest.py
new file mode 100644
index 000000000..b3c150b1a
--- /dev/null
+++ b/tests/test_command_genrequest.py
@@ -0,0 +1,105 @@
+from __future__ import annotations
+
+import ast
+import shutil
+import subprocess
+import sys
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from scrapy.commands.genrequest import _format_code, _read_clipboard
+from scrapy.exceptions import UsageError
+from tests.utils.cmdline import proc
+
+
+def _parse_request_call(code: str) -> dict[str, Any]:
+ """Return the keyword arguments of the single ``Request(...)`` call in *code*.
+
+ Optional ruff formatting changes quote style and line wrapping, so tests
+ compare the parsed call rather than the raw text.
+ """
+ call = ast.parse(code, mode="eval").body
+ assert isinstance(call, ast.Call)
+ return {
+ kw.arg: ast.literal_eval(kw.value) for kw in call.keywords if kw.arg is not None
+ }
+
+
+class TestGenRequestCommand:
+ def test_get(self) -> None:
+ _, out, _ = proc("genrequest", "curl http://example.com/")
+ assert _parse_request_call(out) == {
+ "method": "GET",
+ "url": "http://example.com/",
+ }
+
+ def test_post_with_data(self) -> None:
+ _, out, _ = proc("genrequest", "curl -d title=hello https://example.com/post")
+ assert _parse_request_call(out) == {
+ "method": "POST",
+ "url": "https://example.com/post",
+ "body": "title=hello",
+ }
+
+ def test_headers_and_cookies(self) -> None:
+ _, out, _ = proc(
+ "genrequest",
+ "curl -H 'X-Test: 1' -b 'a=1' https://example.com/",
+ )
+ assert _parse_request_call(out) == {
+ "method": "GET",
+ "url": "https://example.com/",
+ "headers": [("X-Test", "1")],
+ "cookies": {"a": "1"},
+ }
+
+ def test_too_many_arguments(self) -> None:
+ returncode, out, _ = proc("genrequest", "curl http://example.com/", "extra")
+ assert returncode == 2
+ assert "Usage" in out
+
+ def test_invalid_curl_command(self) -> None:
+ returncode, _, err = proc("genrequest", "not a curl command")
+ assert returncode == 2
+ assert "must start with" in err
+
+
+class TestReadClipboard:
+ def test_pyperclip(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setitem(
+ sys.modules, "pyperclip", SimpleNamespace(paste=lambda: "curl a.example ")
+ )
+ assert _read_clipboard() == "curl a.example"
+
+ def test_pyperclip_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setitem(sys.modules, "pyperclip", None)
+ with pytest.raises(UsageError, match="scrapy\\[clipboard\\]"):
+ _read_clipboard()
+
+
+class TestFormatCode:
+ CODE = "Request(method='GET', url='http://example.com/')"
+
+ def test_ruff_not_installed(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(shutil, "which", lambda name: None)
+ assert _format_code(self.CODE) == self.CODE
+
+ def test_ruff_formats(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/ruff")
+ monkeypatch.setattr(
+ subprocess,
+ "run",
+ lambda *a, **k: SimpleNamespace(stdout='Request(url="x")\n'),
+ )
+ assert _format_code(self.CODE) == 'Request(url="x")\n'
+
+ def test_ruff_fails(self, monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(shutil, "which", lambda name: "/usr/bin/ruff")
+
+ def _raise(*args: Any, **kwargs: Any) -> None:
+ raise subprocess.CalledProcessError(1, "ruff")
+
+ monkeypatch.setattr(subprocess, "run", _raise)
+ assert _format_code(self.CODE) == self.CODE
diff --git a/tests/test_commands.py b/tests/test_commands.py
index 3a8b05203..db07a3d4d 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -937,6 +937,7 @@ class TestHelpMessage(TestProjectBase):
"runspider",
"version",
"genspider",
+ "genrequest",
"check",
"bench",
],