Scrapy, a fast high-level web crawling & scraping framework for Python.
FormRequest: handle whitespaces in action attribute properly
Mikhail Korobov074caf434e25
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.pyindex 2862dc096..905d8412f 100644--- a/scrapy/http/request/form.py+++ b/scrapy/http/request/form.py@@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient classSee documentation in docs/topics/request-response.rst"""+import sixfrom six.moves.urllib.parse import urljoin, urlencode+import lxml.htmlfrom parsel.selector import create_root_node-import six+from w3lib.html import strip_html5_whitespace+from scrapy.http.request import Requestfrom scrapy.utils.python import to_bytes, is_listlikefrom scrapy.utils.response import get_base_url@@ -51,7 +54,10 @@ class FormRequest(Request):def _get_form_url(form, url):if url is None:- return urljoin(form.base_url, form.action)+ action = form.get('action')+ if action is None:+ return form.base_url+ return urljoin(form.base_url, strip_html5_whitespace(action))return urljoin(form.base_url, url)diff --git a/tests/test_http_request.py b/tests/test_http_request.pyindex d7216e1d2..7eadb874f 100644--- a/tests/test_http_request.py+++ b/tests/test_http_request.py@@ -556,7 +556,6 @@ class FormRequestTest(RequestTest):fs = _qs(req, to_unicode=True, encoding='latin1')self.assertTrue(fs[u'price in \u00a5'])-def test_from_response_multiple_forms_clickdata(self):response = _buildresponse("""<form name="form1">@@ -989,7 +988,7 @@ class FormRequestTest(RequestTest):"""<html><head>- <base href="http://b.com/">+ <base href=" http://b.com/"></head><body><form action="test_form">@@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest):req = self.request_class.from_response(response)self.assertEqual(req.url, 'http://b.com/test_form')+ def test_spaces_in_action(self):+ resp = _buildresponse('<body><form action=" path\n"></form></body>')+ req = self.request_class.from_response(resp)+ self.assertEqual(req.url, 'http://example.com/path')+def test_from_response_css(self):response = _buildresponse("""<form action="post.php" method="POST">@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest):self.assertRaises(ValueError, self.request_class.from_response,response, formcss="input[name='abc']")+def _buildresponse(body, **kwargs):kwargs.setdefault('body', body)kwargs.setdefault('url', 'http://example.com')kwargs.setdefault('encoding', 'utf-8')return HtmlResponse(**kwargs)+def _qs(req, encoding='utf-8', to_unicode=False):if req.method == 'POST':qs = req.body