Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Fix #3247: Allow scrapy.FormRequest.from_response method to handle duplicate keys

CCInCharge782f866572d8
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 95b38e990..c2413b431 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -114,10 +114,12 @@ def _get_form(response, formname, formid, formnumber, formxpath):
def _get_inputs(form, formdata, dont_click, clickdata, response):
try:
- formdata = dict(formdata or ())
+ formdata_keys = dict(formdata or ()).keys()
except (ValueError, TypeError):
raise ValueError('formdata should be a dict or iterable of tuples')
+ if not formdata:
+ formdata = ()
inputs = form.xpath('descendant::textarea'
'|descendant::select'
'|descendant::input[not(@type) or @type['
@@ -128,14 +130,17 @@ def _get_inputs(form, formdata, dont_click, clickdata, response):
"re": "http://exslt.org/regular-expressions"})
values = [(k, u'' if v is None else v)
for k, v in (_value(e) for e in inputs)
- if k and k not in formdata]
+ if k and k not in formdata_keys]
if not dont_click:
clickable = _get_clickable(clickdata, form)
if clickable and clickable[0] not in formdata and not clickable[0] is None:
values.append(clickable)
- values.extend((k, v) for k, v in formdata.items() if v is not None)
+ if isinstance(formdata, dict):
+ formdata = formdata.items()
+
+ values.extend((k, v) for k, v in formdata if v is not None)
return values
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index a042f03b6..18fc6413c 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -401,6 +401,29 @@ class FormRequestTest(RequestTest):
self.assertEqual(fs[u'test2'], [u'xxx µ'])
self.assertEqual(fs[u'six'], [u'seven'])
+ def test_from_response_duplicate_form_key(self):
+ response = _buildresponse(
+ '<form></form>',
+ url='http://www.example.com')
+ req = self.request_class.from_response(response,
+ method='GET',
+ formdata=(('foo', 'bar'), ('foo', 'baz')))
+ self.assertEqual(urlparse(req.url).hostname, 'www.example.com')
+ self.assertEqual(urlparse(req.url).query, 'foo=bar&foo=baz')
+
+ def test_from_response_override_duplicate_form_key(self):
+ response = _buildresponse(
+ """<form action="get.php" method="POST">
+ <input type="hidden" name="one" value="1">
+ <input type="hidden" name="two" value="3">
+ </form>""")
+ req = self.request_class.from_response(
+ response,
+ formdata=(('two', '2'), ('two', '4')))
+ fs = _qs(req)
+ self.assertEqual(fs[b'one'], [b'1'])
+ self.assertEqual(fs[b'two'], [b'2', b'4'])
+
def test_from_response_extra_headers(self):
response = _buildresponse(
"""<form action="post.php" method="POST">