Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Deprecate FormRequest in favor of form2request (#6438)

Adrián Chaves8d69a7c8659f
diff --git a/docs/conf.py b/docs/conf.py
index e3e618860..99d5df7da 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -154,6 +154,7 @@ scrapy_intersphinx_enable = [
"coverage",
"cryptography",
"cssselect",
+ "form2request",
"itemloaders",
"parsel",
"pytest",
diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rst
index 090caa6e0..5a399e094 100644
--- a/docs/topics/dynamic-content.rst
+++ b/docs/topics/dynamic-content.rst
@@ -83,7 +83,7 @@ request with Scrapy.
It might be enough to yield a :class:`~scrapy.Request` with the same HTTP
method and URL. However, you may also need to reproduce the body, headers and
-form parameters (see :class:`~scrapy.FormRequest`) of that request.
+form parameters (see :ref:`form`) of that request.
As all major browsers allow to export the requests in curl_ format, Scrapy
incorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalent
diff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rst
index ff05cd284..0913a3310 100644
--- a/docs/topics/leaks.rst
+++ b/docs/topics/leaks.rst
@@ -82,7 +82,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:
ExampleSpider 1 oldest: 15s ago
HtmlResponse 10 oldest: 1s ago
Selector 2 oldest: 0s ago
- FormRequest 878 oldest: 7s ago
+ Request 878 oldest: 7s ago
As you can see, that report also shows the "age" of the oldest object in each
class. If you're running multiple spiders per process chances are you can
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index f23c7611c..ba7b3ee3c 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -246,6 +246,78 @@ Request objects
.. automethod:: to_dict
+.. _form:
+
+Creating requests that submit HTML forms
+----------------------------------------
+
+Use :doc:`form2request <form2request:index>` to build request data from an HTML
+``<form>`` element and convert it to a :class:`~scrapy.Request`.
+
+Install it with pip:
+
+.. code-block:: bash
+
+ pip install form2request
+
+Select the desired form with CSS or XPath, then build and convert request
+data:
+
+.. code-block:: python
+
+ from form2request import form2request
+
+
+ def parse(self, response):
+ form = response.css("form#search")
+ request_data = form2request(form, data={"q": "scrapy"})
+ yield request_data.to_scrapy(callback=self.parse_results)
+
+Use ``data`` to override field values. To drop a field from the resulting
+request, set its value to ``None``.
+
+By default, form2request simulates clicking the first submit button. To submit
+without clicking any button, pass ``click=False``. To click a specific submit
+button, pass its element:
+
+.. code-block:: python
+
+ def parse(self, response):
+ form = response.css("form#checkout")
+ submit = form.css('button[name="pay"]')
+ request_data = form2request(form, click=submit)
+
+.. _topics-request-response-ref-request-userlogin:
+
+Using form2request to simulate a user login
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+It is usual for web sites to provide pre-populated form fields through ``<input
+type="hidden">`` elements, such as session related data or authentication
+tokens (for login pages). Build the request from the form and only override the
+credentials:
+
+.. code-block:: python
+
+ import scrapy
+ from form2request import form2request
+
+
+ class LoginSpider(scrapy.Spider):
+ name = "example.com"
+ start_urls = ["http://www.example.com/users/login.php"]
+
+ def parse(self, response):
+ form = response.css("form")
+ request_data = form2request(
+ form,
+ data={"username": "john", "password": "secret"},
+ )
+ yield request_data.to_scrapy(callback=self.after_login)
+
+ def after_login(self, response): ...
+
+
Other functions related to requests
-----------------------------------
@@ -771,159 +843,6 @@ Request subclasses
Here is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclass
it to implement your own custom functionality.
-FormRequest objects
--------------------
-
-The FormRequest class extends the base :class:`~scrapy.Request` with functionality for
-dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form
-fields with form data from :class:`Response` objects.
-
-.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms
-
-.. currentmodule:: None
-
-.. class:: scrapy.FormRequest(url, [formdata, ...])
- :canonical: scrapy.http.request.form.FormRequest
-
- The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The
- remaining arguments are the same as for the :class:`~scrapy.Request` class and are
- not documented here.
-
- :param formdata: is a dictionary (or iterable of (key, value) tuples)
- containing HTML Form data which will be url-encoded and assigned to the
- body of the request.
- :type formdata: dict or collections.abc.Iterable
-
- The :class:`~scrapy.FormRequest` objects support the following class method in
- addition to the standard :class:`~scrapy.Request` methods:
-
- .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
-
- Returns a new :class:`~scrapy.FormRequest` object with its form field values
- pre-populated with those found in the HTML ``<form>`` element contained
- in the given response. For an example see
- :ref:`topics-request-response-ref-request-userlogin`.
-
- The policy is to automatically simulate a click, by default, on any form
- control that looks clickable, like a ``<input type="submit">``. Even
- though this is quite convenient, and often the desired behaviour,
- sometimes it can cause problems which could be hard to debug. For
- example, when working with forms that are filled and/or submitted using
- javascript, the default :meth:`from_response` behaviour may not be the
- most appropriate. To disable this behaviour you can set the
- ``dont_click`` argument to ``True``. Also, if you want to change the
- control clicked (instead of disabling it) you can also use the
- ``clickdata`` argument.
-
- .. caution:: Using this method with select elements which have leading
- or trailing whitespace in the option values will not work due to a
- `bug in lxml`_, which should be fixed in lxml 3.8 and above.
-
- :param response: the response containing a HTML form which will be used
- to pre-populate the form fields
- :type response: :class:`~scrapy.http.Response` object
-
- :param formname: if given, the form with name attribute set to this value will be used.
- :type formname: str
-
- :param formid: if given, the form with id attribute set to this value will be used.
- :type formid: str
-
- :param formxpath: if given, the first form that matches the xpath will be used.
- :type formxpath: str
-
- :param formcss: if given, the first form that matches the css selector will be used.
- :type formcss: str
-
- :param formnumber: the number of form to use, when the response contains
- multiple forms. The first one (and also the default) is ``0``.
- :type formnumber: int
-
- :param formdata: fields to override in the form data. If a field was
- already present in the response ``<form>`` element, its value is
- overridden by the one passed in this parameter. If a value passed in
- this parameter is ``None``, the field will not be included in the
- request, even if it was present in the response ``<form>`` element.
- :type formdata: dict
-
- :param clickdata: attributes to lookup the control clicked. If it's not
- given, the form data will be submitted simulating a click on the
- first clickable element. In addition to html attributes, the control
- can be identified by its zero-based index relative to other
- submittable inputs inside the form, via the ``nr`` attribute.
- :type clickdata: dict
-
- :param dont_click: If True, the form data will be submitted without
- clicking in any element.
- :type dont_click: bool
-
- The other parameters of this class method are passed directly to the
- :class:`~scrapy.FormRequest` ``__init__()`` method.
-
-.. currentmodule:: scrapy.http
-
-Request usage examples
-----------------------
-
-Using FormRequest to send data via HTTP POST
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-If you want to simulate a HTML Form POST in your spider and send a couple of
-key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your
-spider) like this:
-
-.. skip: next
-.. code-block:: python
-
- return [
- FormRequest(
- url="http://www.example.com/post/action",
- formdata={"name": "John Doe", "age": "27"},
- callback=self.after_post,
- )
- ]
-
-.. _topics-request-response-ref-request-userlogin:
-
-Using FormRequest.from_response() to simulate a user login
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-It is usual for web sites to provide pre-populated form fields through ``<input
-type="hidden">`` elements, such as session related data or authentication
-tokens (for login pages). When scraping, you'll want these fields to be
-automatically pre-populated and only override a couple of them, such as the
-user name and password. You can use the :meth:`.FormRequest.from_response`
-method for this job. Here's an example spider which uses it:
-
-.. code-block:: python
-
- import scrapy
-
-
- def authentication_failed(response):
- # TODO: Check the contents of the response and return True if it failed
- # or False if it succeeded.
- pass
-
-
- class LoginSpider(scrapy.Spider):
- name = "example.com"
- start_urls = ["http://www.example.com/users/login.php"]
-
- def parse(self, response):
- return scrapy.FormRequest.from_response(
- response,
- formdata={"username": "john", "password": "secret"},
- callback=self.after_login,
- )
-
- def after_login(self, response):
- if authentication_failed(response):
- self.logger.error("Login failed")
- return
-
- # continue scraping with authenticated session...
-
JsonRequest
-----------
diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.py
index 0e5c2b53b..e20e894ae 100644
--- a/scrapy/http/__init__.py
+++ b/scrapy/http/__init__.py
@@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers,
Request and Response outside this module.
"""
+from warnings import catch_warnings, filterwarnings
+
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.headers import Headers
from scrapy.http.request import Request
-from scrapy.http.request.form import FormRequest
from scrapy.http.request.json_request import JsonRequest
from scrapy.http.request.rpc import XmlRpcRequest
from scrapy.http.response import Response
@@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponse
from scrapy.http.response.json import JsonResponse
from scrapy.http.response.text import TextResponse
from scrapy.http.response.xml import XmlResponse
+from scrapy.utils.deprecate import create_deprecated_class
+
+with catch_warnings():
+ filterwarnings("ignore", category=ScrapyDeprecationWarning)
+
+ from scrapy.http.request.form import FormRequest as _FormRequest
+
+ FormRequest = create_deprecated_class(
+ name="FormRequest",
+ new_class=_FormRequest,
+ subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",
+ instance_warn_message="{cls} is deprecated, use the form2request library instead.",
+ )
__all__ = [
"FormRequest",
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index affa14499..4da595b22 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -10,10 +10,12 @@ from __future__ import annotations
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, cast
from urllib.parse import urlencode, urljoin, urlsplit, urlunsplit
+from warnings import warn
from parsel.csstranslator import HTMLTranslator
from w3lib.html import strip_html5_whitespace
+from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.http.request import Request
from scrapy.utils.python import is_listlike, to_bytes
@@ -30,6 +32,12 @@ if TYPE_CHECKING:
from scrapy.http.response.text import TextResponse
+warn(
+ "The entire scrapy.http.request.form module is deprecated. Use the "
+ "form2request library instead.",
+ ScrapyDeprecationWarning,
+ stacklevel=2,
+)
FormdataVType: TypeAlias = str | Iterable[str]
FormdataKVType: TypeAlias = tuple[str, FormdataVType]
diff --git a/tests/test_contracts.py b/tests/test_contracts.py
index a35ef7010..008e326ec 100644
--- a/tests/test_contracts.py
+++ b/tests/test_contracts.py
@@ -3,7 +3,6 @@ from unittest import TextTestResult
import pytest
from twisted.python import failure
-from scrapy import FormRequest
from scrapy.contracts import Contract, ContractsManager
from scrapy.contracts.default import (
CallbackKeywordArgumentsContract,
@@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock):
meta = None
+class TaggedRequest(Request):
+ def __init__(self, url, contract_tag=None, **kwargs):
+ super().__init__(url, **kwargs)
+ self.contract_tag = contract_tag
+
+
class CustomSuccessContract(Contract):
name = "custom_success_contract"
@@ -49,12 +54,13 @@ class CustomFailContract(Contract):
raise TypeError("Error in adjust_request_args")
-class CustomFormContract(Contract):
- name = "custom_form"
- request_cls = FormRequest
+class CustomTaggedRequestContract(Contract):
+ name = "custom_tagged_request"
+ request_cls = TaggedRequest
def adjust_request_args(self, args):
- args["formdata"] = {"name": "scrapy"}
+ args["contract_tag"] = "custom"
+ args["method"] = "POST"
return args
@@ -179,10 +185,10 @@ class DemoSpider(Spider):
@returns items 1 1
"""
- def custom_form(self, response):
+ def custom_tagged_request(self, response):
"""
@url http://scrapy.org
- @custom_form
+ @custom_tagged_request
"""
def invalid_regex(self, response):
@@ -253,7 +259,7 @@ class TestContractsManager:
MetadataContract,
ReturnsContract,
ScrapesContract,
- CustomFormContract,
+ CustomTaggedRequestContract,
CustomSuccessContract,
CustomFailContract,
]
@@ -533,17 +539,21 @@ class TestContractsManager:
assert crawler.spider.visited == 2
- def test_form_contract(self):
+ def test_custom_tagged_request_contract(self):
spider = DemoSpider()
- request = self.conman.from_method(spider.custom_form, self.results)
+ request = self.conman.from_method(spider.custom_tagged_request, self.results)
assert request.method == "POST"
- assert isinstance(request, FormRequest)
+ assert isinstance(request, TaggedRequest)
+ assert request.contract_tag == "custom"
def test_inherited_contracts(self):
spider = InheritsDemoSpider()
requests = self.conman.from_spider(spider, self.results)
assert requests
+ assert any(
+ isinstance(request, TaggedRequest) for request in requests if request
+ )
class CustomFailContractPreProcess(Contract):
diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.py
index 92f2f3fac..8ba9f3422 100644
--- a/tests/test_http_request_form.py
+++ b/tests/test_http_request_form.py
@@ -27,7 +27,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):
class TestFormRequest(TestRequest):
- request_class = FormRequest
+ request_class = FormRequest # type: ignore[assignment]
def assertQueryEqual(self, first, second, msg=None):
first = to_unicode(first).split("&")
diff --git a/tests/test_request_dict.py b/tests/test_request_dict.py
index ea7018541..78ff18b15 100644
--- a/tests/test_request_dict.py
+++ b/tests/test_request_dict.py
@@ -1,7 +1,7 @@
import pytest
from scrapy import Request, Spider
-from scrapy.http import FormRequest, JsonRequest
+from scrapy.http import JsonRequest
from scrapy.utils.request import request_from_dict
@@ -67,12 +67,10 @@ class TestRequestSerialization:
assert r1.dumps_kwargs == r2.dumps_kwargs
def test_request_class(self):
- r1 = FormRequest("http://www.example.com")
+ r1 = CustomRequest("http://www.example.com")
self._assert_serializes_ok(r1, spider=self.spider)
- r2 = CustomRequest("http://www.example.com")
+ r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
self._assert_serializes_ok(r2, spider=self.spider)
- r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})
- self._assert_serializes_ok(r3, spider=self.spider)
def test_callback_serialization(self):
r = Request(
diff --git a/tox.ini b/tox.ini
index 219a2b616..f1b004ab8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -245,6 +245,7 @@ changedir = docs
deps =
{[test-requirements]deps}
-rdocs/requirements.txt
+ form2request
commands =
pytest