Scrapy, a fast high-level web crawling & scraping framework for Python.
implements FormRequest.from_response CSS support
Capi Etheriel659715ecd92c
diff --git a/docs/topics/request-response.rst b/docs/topics/request-response.rstindex 0abec1f96..8f519b459 100644--- a/docs/topics/request-response.rst+++ b/docs/topics/request-response.rst@@ -282,7 +282,7 @@ fields with form data from :class:`Response` objects.The :class:`FormRequest` objects support the following class method inaddition to the standard :class:`Request` methods:- .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, clickdata=None, dont_click=False, ...])+ .. classmethod:: FormRequest.from_response(response, [formname=None, formnumber=0, formdata=None, formxpath=None, formcss=None, clickdata=None, dont_click=False, ...])Returns a new :class:`FormRequest` object with its form field valuespre-populated with those found in the HTML ``<form>`` element contained@@ -310,6 +310,9 @@ fields with form data from :class:`Response` objects.:param formxpath: if given, the first form that matches the xpath will be used.:type formxpath: string+ :param formcss: if given, the first form that matches the css selector will be used.+ :type formcss: string+:param formnumber: the number of form to use, when the response containsmultiple forms. The first one (and also the default) is ``0``.:type formnumber: integer@@ -339,6 +342,9 @@ fields with form data from :class:`Response` objects... versionadded:: 0.17The ``formxpath`` parameter.+ .. versionadded:: 1.1.5+ The ``formcss`` parameter.+Request usage examples----------------------diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex f623a5aa3..5501634d3 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -34,8 +34,14 @@ class FormRequest(Request):@classmethoddef from_response(cls, response, formname=None, formid=None, formnumber=0, formdata=None,- clickdata=None, dont_click=False, formxpath=None, **kwargs):+ clickdata=None, dont_click=False, formxpath=None, formcss=None, **kwargs):+kwargs.setdefault('encoding', response.encoding)++ if formcss is not None:+ from parsel.csstranslator import HTMLTranslator+ formxpath = HTMLTranslator().css_to_xpath(formcss)+form = _get_form(response, formname, formid, formnumber, formxpath)formdata = _get_inputs(form, formdata, dont_click, clickdata, response)url = _get_form_url(form, kwargs.pop('url', None))@@ -73,7 +79,7 @@ def _get_form(response, formname, formid, formnumber, formxpath):f = root.xpath('//form[@id="%s"]' % formid)if f:return f[0]-+# Get form element from xpath, if not found, go upif formxpath is not None:nodes = root.xpath(formxpath)diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex f82b2de8d..b81d43c41 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -846,6 +846,26 @@ class FormRequestTest(RequestTest):req = self.request_class.from_response(response)self.assertEqual(req.url, 'http://b.com/test_form')+ def test_from_response_css(self):+ response = _buildresponse(+ """<form action="post.php" method="POST">+ <input type="hidden" name="one" value="1">+ <input type="hidden" name="two" value="2">+ </form>+ <form action="post2.php" method="POST">+ <input type="hidden" name="three" value="3">+ <input type="hidden" name="four" value="4">+ </form>""")+ r1 = self.request_class.from_response(response, formcss="form[action='post.php']")+ fs = _qs(r1)+ self.assertEqual(fs[b'one'], [b'1'])++ r1 = self.request_class.from_response(response, formcss="input[name='four']")+ fs = _qs(r1)+ self.assertEqual(fs[b'three'], [b'3'])++ self.assertRaises(ValueError, self.request_class.from_response,+ response, formcss="input[name='abc']")def _buildresponse(body, **kwargs):kwargs.setdefault('body', body)