Scrapy, a fast high-level web crawling & scraping framework for Python.
Fix #3247: Allow scrapy.FormRequest.from_response method to handle duplicate keys
CCInCharge782f866572d8
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 95b38e990..c2413b431 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):def _get_inputs(form, formdata, dont_click, clickdata, response):try:- formdata = dict(formdata or ())+ formdata_keys = dict(formdata or ()).keys()except (ValueError, TypeError):raise ValueError('formdata should be a dict or iterable of tuples')+ if not formdata:+ formdata = ()inputs = form.xpath('descendant::textarea''|descendant::select''|descendant::input[not(@type) or @type['@@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):"re": "http://exslt.org/regular-expressions"})values = [(k, u'' if v is None else v)for k, v in (_value(e) for e in inputs)- if k and k not in formdata]+ if k and k not in formdata_keys]if not dont_click:clickable = _get_clickable(clickdata, form)if clickable and clickable[0] not in formdata and not clickable[0] is None:values.append(clickable)- values.extend((k, v) for k, v in formdata.items() if v is not None)+ if isinstance(formdata, dict):+ formdata = formdata.items()++ values.extend((k, v) for k, v in formdata if v is not None)return valuesdiff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex a042f03b6..18fc6413c 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -401,6 +401,29 @@ class FormRequestTest(RequestTest):self.assertEqual(fs[u'test2'], [u'xxx µ'])self.assertEqual(fs[u'six'], [u'seven'])+ def test_from_response_duplicate_form_key(self):+ response = _buildresponse(+ '<form></form>',+ url='http://www.example.com')+ req = self.request_class.from_response(response,+ method='GET',+ formdata=(('foo', 'bar'), ('foo', 'baz')))+ self.assertEqual(urlparse(req.url).hostname, 'www.example.com')+ self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')++ def test_from_response_override_duplicate_form_key(self):+ response = _buildresponse(+ """<form action="get.php" method="POST">+ <input type="hidden" name="one" value="1">+ <input type="hidden" name="two" value="3">+ </form>""")+ req = self.request_class.from_response(+ response,+ formdata=(('two', '2'), ('two', '4')))+ fs = _qs(req)+ self.assertEqual(fs[b'one'], [b'1'])+ self.assertEqual(fs[b'two'], [b'2', b'4'])+def test_from_response_extra_headers(self):response = _buildresponse("""<form action="post.php" method="POST">