Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

implements FormRequest.from_response CSS support

Capi Etheriel659715ecd92c
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rst
index 0abec1f96..8f519b459 100644
--- a/docs/topics/request-response.rst
+++ b/docs/topics/request-response.rst
@@ -282,7 +282,7 @@ fields with form data from :class:`Response` objects.
The :class:`FormRequest` objects support the following class method in
addition to the standard :class:`Request` methods:
- .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, clickdata=None, dont_click=False, ...])
+ .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])
Returns a new :class:`FormRequest` object with its form field values
pre-populated with those found in the HTML ``<form>`` element contained
@@ -310,6 +310,9 @@ fields with form data from :class:`Response` objects.
:param formxpath: if given, the first form that matches the xpath will be used.
:type formxpath: string
+ :param formcss: if given, the first form that matches the css selector will be used.
+ :type formcss: string
+
:param formnumber: the number of form to use, when the response contains
multiple forms. The first one (and also the default) is ``0``.
:type formnumber: integer
@@ -339,6 +342,9 @@ fields with form data from :class:`Response` objects.
.. versionadded:: 0.17
The ``formxpath`` parameter.
+ .. versionadded:: 1.1.5
+ The ``formcss`` parameter.
+
Request usage examples
----------------------
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index f623a5aa3..5501634d3 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -34,8 +34,14 @@ class FormRequest(Request):
@classmethod
def from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,
- clickdata=None, dont_click=False, formxpath=None, **kwargs):
+ clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):
+
kwargs.setdefault('encoding', response.encoding)
+
+ if formcss is not None:
+ from parsel.csstranslator import HTMLTranslator
+ formxpath = HTMLTranslator().css_to_xpath(formcss)
+
form = _get_form(response, formname, formid, formnumber, formxpath)
formdata = _get_inputs(form, formdata, dont_click, clickdata, response)
url = _get_form_url(form, kwargs.pop('url', None))
@@ -73,7 +79,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):
f = root.xpath('//form[@id="%s"]' % formid)
if f:
return f[0]
-
+
# Get form element from xpath, if not found, go up
if formxpath is not None:
nodes = root.xpath(formxpath)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index f82b2de8d..b81d43c41 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -846,6 +846,26 @@ class FormRequestTest(RequestTest):
req = self.request_class.from_response(response)
self.assertEqual(req.url, 'http://b.com/test_form')
+ def test_from_response_css(self):
+ response = _buildresponse(
+ """<form action="post.php" method="POST">
+ <input type="hidden" name="one" value="1">
+ <input type="hidden" name="two" value="2">
+ </form>
+ <form action="post2.php" method="POST">
+ <input type="hidden" name="three" value="3">
+ <input type="hidden" name="four" value="4">
+ </form>""")
+ r1 = self.request_class.from_response(response, formcss="form[action='post.php']")
+ fs = _qs(r1)
+ self.assertEqual(fs[b'one'], [b'1'])
+
+ r1 = self.request_class.from_response(response, formcss="input[name='four']")
+ fs = _qs(r1)
+ self.assertEqual(fs[b'three'], [b'3'])
+
+ self.assertRaises(ValueError, self.request_class.from_response,
+ response, formcss="input[name='abc']")
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)