Scrapy, a fast high-level web crawling & scraping framework for Python.
Deprecate FormRequest in favor of form2request (#6438)
Adrián Chaves8d69a7c8659f
diff --git a/docs/conf.py b/docs/conf.pyindex e3e618860..99d5df7da 100644--- a/docs/conf.py+++ b/docs/conf.py@@ -154,6 +154,7 @@ scrapy_intersphinx_enable = ["coverage","cryptography","cssselect",+ "form2request","itemloaders","parsel","pytest",diff --git a/docs/topics/dynamic-content.rst b/docs/topics/dynamic-content.rstindex 090caa6e0..5a399e094 100644--- a/docs/topics/dynamic-content.rst+++ b/docs/topics/dynamic-content.rst@@ -83,7 +83,7 @@ request with Scrapy.It might be enough to yield a :class:`~scrapy.Request` with the same HTTPmethod and URL. However, you may also need to reproduce the body, headers and-form parameters (see :class:`~scrapy.FormRequest`) of that request.+form parameters (see :ref:`form`) of that request.As all major browsers allow to export the requests in curl_ format, Scrapyincorporates the method :meth:`~scrapy.Request.from_curl` to generate an equivalentdiff --git a/docs/topics/leaks.rst b/docs/topics/leaks.rstindex ff05cd284..0913a3310 100644--- a/docs/topics/leaks.rst+++ b/docs/topics/leaks.rst@@ -82,7 +82,7 @@ alias to the :func:`~scrapy.utils.trackref.print_live_refs` function:ExampleSpider 1 oldest: 15s agoHtmlResponse 10 oldest: 1s agoSelector 2 oldest: 0s ago- FormRequest 878 oldest: 7s ago+ Request 878 oldest: 7s agoAs you can see, that report also shows the "age" of the oldest object in eachclass. If you're running multiple spiders per process chances are you candiff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex f23c7611c..ba7b3ee3c 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -246,6 +246,78 @@ Request objects.. automethod:: to_dict+.. _form:++Creating requests that submit HTML forms+----------------------------------------++Use :doc:`form2request <form2request:index>` to build request data from an HTML+``<form>`` element and convert it to a :class:`~scrapy.Request`.++Install it with pip:++.. code-block:: bash++ pip install form2request++Select the desired form with CSS or XPath, then build and convert request+data:++.. code-block:: python++ from form2request import form2request+++ def parse(self, response):+ form = response.css("form#search")+ request_data = form2request(form, data={"q": "scrapy"})+ yield request_data.to_scrapy(callback=self.parse_results)++Use ``data`` to override field values. To drop a field from the resulting+request, set its value to ``None``.++By default, form2request simulates clicking the first submit button. To submit+without clicking any button, pass ``click=False``. To click a specific submit+button, pass its element:++.. code-block:: python++ def parse(self, response):+ form = response.css("form#checkout")+ submit = form.css('button[name="pay"]')+ request_data = form2request(form, click=submit)++.. _topics-request-response-ref-request-userlogin:++Using form2request to simulate a user login+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~++It is usual for web sites to provide pre-populated form fields through ``<input+type="hidden">`` elements, such as session related data or authentication+tokens (for login pages). Build the request from the form and only override the+credentials:++.. code-block:: python++ import scrapy+ from form2request import form2request+++ class LoginSpider(scrapy.Spider):+ name = "example.com"+ start_urls = ["http://www.example.com/users/login.php"]++ def parse(self, response):+ form = response.css("form")+ request_data = form2request(+ form,+ data={"username": "john", "password": "secret"},+ )+ yield request_data.to_scrapy(callback=self.after_login)++ def after_login(self, response): ...++Other functions related to requests-----------------------------------@@ -771,159 +843,6 @@ Request subclassesHere is the list of built-in :class:`~scrapy.Request` subclasses. You can also subclassit to implement your own custom functionality.-FormRequest objects----------------------The FormRequest class extends the base :class:`~scrapy.Request` with functionality for-dealing with HTML forms. It uses `lxml.html forms`_ to pre-populate form-fields with form data from :class:`Response` objects.--.. _lxml.html forms: https://lxml.de/lxmlhtml.html#forms--.. currentmodule:: None--.. class:: scrapy.FormRequest(url, [formdata, ...])- :canonical: scrapy.http.request.form.FormRequest-- The :class:`~scrapy.FormRequest` class adds a new keyword parameter to the ``__init__()`` method. The- remaining arguments are the same as for the :class:`~scrapy.Request` class and are- not documented here.-- :param formdata: is a dictionary (or iterable of (key, value) tuples)- containing HTML Form data which will be url-encoded and assigned to the- body of the request.- :type formdata: dict or collections.abc.Iterable-- The :class:`~scrapy.FormRequest` objects support the following class method in- addition to the standard :class:`~scrapy.Request` methods:-- .. classmethod:: from_response(response, [formname=None, formid=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])-- Returns a new :class:`~scrapy.FormRequest` object with its form field values- pre-populated with those found in the HTML ``<form>`` element contained- in the given response. For an example see- :ref:`topics-request-response-ref-request-userlogin`.-- The policy is to automatically simulate a click, by default, on any form- control that looks clickable, like a ``<input type="submit">``. Even- though this is quite convenient, and often the desired behaviour,- sometimes it can cause problems which could be hard to debug. For- example, when working with forms that are filled and/or submitted using- javascript, the default :meth:`from_response` behaviour may not be the- most appropriate. To disable this behaviour you can set the- ``dont_click`` argument to ``True``. Also, if you want to change the- control clicked (instead of disabling it) you can also use the- ``clickdata`` argument.-- .. caution:: Using this method with select elements which have leading- or trailing whitespace in the option values will not work due to a- `bug in lxml`_, which should be fixed in lxml 3.8 and above.-- :param response: the response containing a HTML form which will be used- to pre-populate the form fields- :type response: :class:`~scrapy.http.Response` object-- :param formname: if given, the form with name attribute set to this value will be used.- :type formname: str-- :param formid: if given, the form with id attribute set to this value will be used.- :type formid: str-- :param formxpath: if given, the first form that matches the xpath will be used.- :type formxpath: str-- :param formcss: if given, the first form that matches the css selector will be used.- :type formcss: str-- :param formnumber: the number of form to use, when the response contains- multiple forms. The first one (and also the default) is ``0``.- :type formnumber: int-- :param formdata: fields to override in the form data. If a field was- already present in the response ``<form>`` element, its value is- overridden by the one passed in this parameter. If a value passed in- this parameter is ``None``, the field will not be included in the- request, even if it was present in the response ``<form>`` element.- :type formdata: dict-- :param clickdata: attributes to lookup the control clicked. If it's not- given, the form data will be submitted simulating a click on the- first clickable element. In addition to html attributes, the control- can be identified by its zero-based index relative to other- submittable inputs inside the form, via the ``nr`` attribute.- :type clickdata: dict-- :param dont_click: If True, the form data will be submitted without- clicking in any element.- :type dont_click: bool-- The other parameters of this class method are passed directly to the- :class:`~scrapy.FormRequest` ``__init__()`` method.--.. currentmodule:: scrapy.http--Request usage examples-------------------------Using FormRequest to send data via HTTP POST-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~--If you want to simulate a HTML Form POST in your spider and send a couple of-key-value fields, you can return a :class:`~scrapy.FormRequest` object (from your-spider) like this:--.. skip: next-.. code-block:: python-- return [- FormRequest(- url="http://www.example.com/post/action",- formdata={"name": "John Doe", "age": "27"},- callback=self.after_post,- )- ]--.. _topics-request-response-ref-request-userlogin:--Using FormRequest.from_response() to simulate a user login-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~--It is usual for web sites to provide pre-populated form fields through ``<input-type="hidden">`` elements, such as session related data or authentication-tokens (for login pages). When scraping, you'll want these fields to be-automatically pre-populated and only override a couple of them, such as the-user name and password. You can use the :meth:`.FormRequest.from_response`-method for this job. Here's an example spider which uses it:--.. code-block:: python-- import scrapy--- def authentication_failed(response):- # TODO: Check the contents of the response and return True if it failed- # or False if it succeeded.- pass--- class LoginSpider(scrapy.Spider):- name = "example.com"- start_urls = ["http://www.example.com/users/login.php"]-- def parse(self, response):- return scrapy.FormRequest.from_response(- response,- formdata={"username": "john", "password": "secret"},- callback=self.after_login,- )-- def after_login(self, response):- if authentication_failed(response):- self.logger.error("Login failed")- return-- # continue scraping with authenticated session...-JsonRequest-----------diff --git a/scrapy/http/__init__.py b/scrapy/http/__init__.pyindex 0e5c2b53b..e20e894ae 100644--- a/scrapy/http/__init__.py+++ b/scrapy/http/__init__.py@@ -5,9 +5,11 @@ Use this module (instead of the more specific ones) when importing Headers,Request and Response outside this module."""+from warnings import catch_warnings, filterwarnings++from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.headers import Headersfrom scrapy.http.request import Request-from scrapy.http.request.form import FormRequestfrom scrapy.http.request.json_request import JsonRequestfrom scrapy.http.request.rpc import XmlRpcRequestfrom scrapy.http.response import Response@@ -15,6 +17,19 @@ from scrapy.http.response.html import HtmlResponsefrom scrapy.http.response.json import JsonResponsefrom scrapy.http.response.text import TextResponsefrom scrapy.http.response.xml import XmlResponse+from scrapy.utils.deprecate import create_deprecated_class++with catch_warnings():+ filterwarnings("ignore", category=ScrapyDeprecationWarning)++ from scrapy.http.request.form import FormRequest as _FormRequest++ FormRequest = create_deprecated_class(+ name="FormRequest",+ new_class=_FormRequest,+ subclass_warn_message="{cls} inherits from deprecated class {old}, use the form2request library instead.",+ instance_warn_message="{cls} is deprecated, use the form2request library instead.",+ )__all__ = ["FormRequest",diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex affa14499..4da595b22 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -10,10 +10,12 @@ from __future__ import annotationsfrom collections.abc import Iterablefrom typing import TYPE_CHECKING, Any, ClassVar, TypeAlias, castfrom urllib.parse import urlencode, urljoin, urlsplit, urlunsplit+from warnings import warnfrom parsel.csstranslator import HTMLTranslatorfrom w3lib.html import strip_html5_whitespace+from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http.request import Requestfrom scrapy.utils.python import is_listlike, to_bytes@@ -30,6 +32,12 @@ if TYPE_CHECKING:from scrapy.http.response.text import TextResponse+warn(+ "The entire scrapy.http.request.form module is deprecated. Use the "+ "form2request library instead.",+ ScrapyDeprecationWarning,+ stacklevel=2,+)FormdataVType: TypeAlias = str | Iterable[str]FormdataKVType: TypeAlias = tuple[str, FormdataVType]diff --git a/tests/test_contracts.py b/tests/test_contracts.pyindex a35ef7010..008e326ec 100644--- a/tests/test_contracts.py+++ b/tests/test_contracts.py@@ -3,7 +3,6 @@ from unittest import TextTestResultimport pytestfrom twisted.python import failure-from scrapy import FormRequestfrom scrapy.contracts import Contract, ContractsManagerfrom scrapy.contracts.default import (CallbackKeywordArgumentsContract,@@ -34,6 +33,12 @@ class ResponseMetaMock(ResponseMock):meta = None+class TaggedRequest(Request):+ def __init__(self, url, contract_tag=None, **kwargs):+ super().__init__(url, **kwargs)+ self.contract_tag = contract_tag++class CustomSuccessContract(Contract):name = "custom_success_contract"@@ -49,12 +54,13 @@ class CustomFailContract(Contract):raise TypeError("Error in adjust_request_args")-class CustomFormContract(Contract):- name = "custom_form"- request_cls = FormRequest+class CustomTaggedRequestContract(Contract):+ name = "custom_tagged_request"+ request_cls = TaggedRequestdef adjust_request_args(self, args):- args["formdata"] = {"name": "scrapy"}+ args["contract_tag"] = "custom"+ args["method"] = "POST"return args@@ -179,10 +185,10 @@ class DemoSpider(Spider):@returns items 1 1"""- def custom_form(self, response):+ def custom_tagged_request(self, response):"""@url http://scrapy.org- @custom_form+ @custom_tagged_request"""def invalid_regex(self, response):@@ -253,7 +259,7 @@ class TestContractsManager:MetadataContract,ReturnsContract,ScrapesContract,- CustomFormContract,+ CustomTaggedRequestContract,CustomSuccessContract,CustomFailContract,]@@ -533,17 +539,21 @@ class TestContractsManager:assert crawler.spider.visited == 2- def test_form_contract(self):+ def test_custom_tagged_request_contract(self):spider = DemoSpider()- request = self.conman.from_method(spider.custom_form, self.results)+ request = self.conman.from_method(spider.custom_tagged_request, self.results)assert request.method == "POST"- assert isinstance(request, FormRequest)+ assert isinstance(request, TaggedRequest)+ assert request.contract_tag == "custom"def test_inherited_contracts(self):spider = InheritsDemoSpider()requests = self.conman.from_spider(spider, self.results)assert requests+ assert any(+ isinstance(request, TaggedRequest) for request in requests if request+ )class CustomFailContractPreProcess(Contract):diff --git a/tests/test_http_request_form.py b/tests/test_http_request_form.pyindex 92f2f3fac..8ba9f3422 100644--- a/tests/test_http_request_form.py+++ b/tests/test_http_request_form.py@@ -27,7 +27,7 @@ def _qs(req, encoding="utf-8", to_unicode=False):class TestFormRequest(TestRequest):- request_class = FormRequest+ request_class = FormRequest # type: ignore[assignment]def assertQueryEqual(self, first, second, msg=None):first = to_unicode(first).split("&")diff --git a/tests/test_request_dict.py b/tests/test_request_dict.pyindex ea7018541..78ff18b15 100644--- a/tests/test_request_dict.py+++ b/tests/test_request_dict.py@@ -1,7 +1,7 @@import pytestfrom scrapy import Request, Spider-from scrapy.http import FormRequest, JsonRequest+from scrapy.http import JsonRequestfrom scrapy.utils.request import request_from_dict@@ -67,12 +67,10 @@ class TestRequestSerialization:assert r1.dumps_kwargs == r2.dumps_kwargsdef test_request_class(self):- r1 = FormRequest("http://www.example.com")+ r1 = CustomRequest("http://www.example.com")self._assert_serializes_ok(r1, spider=self.spider)- r2 = CustomRequest("http://www.example.com")+ r2 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})self._assert_serializes_ok(r2, spider=self.spider)- r3 = JsonRequest("http://www.example.com", dumps_kwargs={"indent": 4})- self._assert_serializes_ok(r3, spider=self.spider)def test_callback_serialization(self):r = Request(diff --git a/tox.ini b/tox.iniindex 219a2b616..f1b004ab8 100644--- a/tox.ini+++ b/tox.ini@@ -245,6 +245,7 @@ changedir = docsdeps ={[test-requirements]deps}-rdocs/requirements.txt+ form2requestcommands =pytest