Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Undeprecated the basic FormRequest API (#7671)

Adriancf5607f8bca4
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 700238fe9..5e70a4f98 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -911,6 +911,11 @@ Request subclasses
Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
it to implement your own custom functionality.
+FormRequest
+-----------
+
+.. autoclass:: scrapy.FormRequest
+
JsonRequest
-----------
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index e20e894ae..0e5c2b53b 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -5,11 +5,9 @@ Use this module (instead of the more specific ones) when importing Headers,
Request and Response outside this module.
"""
-from warnings import catch_warnings, filterwarnings
-
-from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.headers import Headers
from scrapy.http.request import Request
+from scrapy.http.request.form import FormRequest
from scrapy.http.request.json_request import JsonRequest
from scrapy.http.request.rpc import XmlRpcRequest
from scrapy.http.response import Response
@@ -17,19 +15,6 @@ from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
from scrapy.http.response.text import TextResponse
from scrapy.http.response.xml import XmlResponse
-from scrapy.utils.deprecate import create_deprecated_class
-
-with catch_warnings():
- filterwarnings("ignore", category=ScrapyDeprecationWarning)
-
- from scrapy.http.request.form import FormRequest as _FormRequest
-
- FormRequest = create_deprecated_class(
- name="FormRequest",
- new_class=_FormRequest,
- subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
- instance_warn_message="{cls} is deprecated, use the form2request library instead.",
- )
__all__ = [
"FormRequest",
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 4da595b22..f1a8dbf3b 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -32,19 +32,61 @@ if TYPE_CHECKING:
from scrapy.http.response.text import TextResponse
-warn(
- "The entire scrapy.http.request.form module is deprecated. Use the "
- "form2request library instead.",
- ScrapyDeprecationWarning,
- stacklevel=2,
-)
-
FormdataVType: TypeAlias = str | Iterable[str]
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
FormdataType: TypeAlias = dict[str, FormdataVType] | list[FormdataKVType] | None
class FormRequest(Request):
+ """A :class:`~scrapy.Request` subclass with a ``formdata`` parameter that
+ url-encodes the given data and assigns it to the request, which makes it
+ convenient to send arbitrary form data via HTTP POST or GET without an HTML
+ ``<form>`` element to parse.
+
+ .. note:: To build a request from an HTML ``<form>`` element found in a
+ response, use :doc:`form2request <form2request:index>` instead. See
+ :ref:`form`.
+
+ The remaining arguments are the same as for the :class:`~scrapy.Request`
+ class and are not documented here.
+
+ :param formdata: a dictionary (or iterable of (key, value) tuples)
+ containing HTML form data which will be url-encoded. If
+ :attr:`~scrapy.Request.method` is not given and ``formdata`` is
+ provided, the method is set to ``"POST"`` and the data is assigned to
+ the request body; if the method is ``"GET"``, the data is added to the
+ URL query string instead.
+ :type formdata: dict or collections.abc.Iterable
+
+ To send data via HTTP POST, simulating an HTML form submission, return a
+ :class:`~scrapy.FormRequest` object from your spider:
+
+ .. skip: next
+ .. code-block:: python
+
+ return [
+ FormRequest(
+ url="http://www.example.com/post/action",
+ formdata={"name": "John Doe", "age": "27"},
+ callback=self.after_post,
+ )
+ ]
+
+ To send the data in the URL query string instead, use the ``GET`` method:
+
+ .. skip: next
+ .. code-block:: python
+
+ return [
+ FormRequest(
+ url="http://www.example.com/search",
+ method="GET",
+ formdata={"q": "keyword", "page": "1"},
+ callback=self.parse_results,
+ )
+ ]
+ """
+
__slots__ = ()
valid_form_methods: ClassVar[list[str]] = ["GET", "POST"]
@@ -84,6 +126,13 @@ class FormRequest(Request):
formcss: str | None = None,
**kwargs: Any,
) -> Self:
+ warn(
+ "FormRequest.from_response() is deprecated. Use the form2request "
+ "library instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+ )
+
kwargs.setdefault("encoding", response.encoding)
if formcss is not None:
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index c667e48a1..af86b35c0 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -1,10 +1,12 @@
from __future__ import annotations
import re
+import warnings
from urllib.parse import parse_qs, unquote_to_bytes
import pytest
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http import FormRequest, HtmlResponse
from scrapy.utils.httpobj import urlparse_cached
from scrapy.utils.python import to_unicode
@@ -26,15 +28,39 @@ def _qs(req, encoding="utf-8", to_unicode=False):
return parse_qs(uqs, True)
+# FormRequest.from_response() is deprecated in favor of form2request, so the
+# many tests below that exercise it ignore the resulting deprecation warning.
@pytest.mark.filterwarnings("ignore::scrapy.exceptions.ScrapyDeprecationWarning")
class TestFormRequest(TestRequest):
- request_class = FormRequest # type: ignore[assignment]
+ request_class = FormRequest
def assertQueryEqual(self, first, second, msg=None):
first = to_unicode(first).split("&")
second = to_unicode(second).split("&")
assert sorted(first) == sorted(second), msg
+ def test_init_not_deprecated(self):
+ # Building a request directly from form data is not deprecated.
+ with warnings.catch_warnings():
+ warnings.simplefilter("error", ScrapyDeprecationWarning)
+ self.request_class(
+ "http://www.example.com", formdata={"a": "1"}, method="POST"
+ )
+ self.request_class(
+ "http://www.example.com", method="GET", formdata={"a": "1"}
+ )
+
+ def test_from_response_deprecated(self):
+ response = _buildresponse(
+ """<form action="post.php" method="POST">
+ <input type="hidden" name="one" value="1">
+ </form>"""
+ )
+ with pytest.warns(
+ ScrapyDeprecationWarning, match=r"FormRequest\.from_response\(\)"
+ ):
+ self.request_class.from_response(response)
+
def test_empty_formdata(self):
r1 = self.request_class("http://www.example.com", formdata={})
assert r1.body == b""