Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

FormRequest: handle whitespaces in action attribute properly

Mikhail Korobov074caf434e25
diff --git a/scrapy/http/request/form.py b/scrapy/http/request/form.py
index 2862dc096..905d8412f 100644
--- a/scrapy/http/request/form.py
+++ b/scrapy/http/request/form.py
@@ -5,10 +5,13 @@ This module implements the FormRequest class which is a more convenient class
See documentation in docs/topics/request-response.rst
"""
+import six
from six.moves.urllib.parse import urljoin, urlencode
+
import lxml.html
from parsel.selector import create_root_node
-import six
+from w3lib.html import strip_html5_whitespace
+
from scrapy.http.request import Request
from scrapy.utils.python import to_bytes, is_listlike
from scrapy.utils.response import get_base_url
@@ -51,7 +54,10 @@ class FormRequest(Request):
def _get_form_url(form, url):
if url is None:
- return urljoin(form.base_url, form.action)
+ action = form.get('action')
+ if action is None:
+ return form.base_url
+ return urljoin(form.base_url, strip_html5_whitespace(action))
return urljoin(form.base_url, url)
diff --git a/tests/test_http_request.py b/tests/test_http_request.py
index d7216e1d2..7eadb874f 100644
--- a/tests/test_http_request.py
+++ b/tests/test_http_request.py
@@ -556,7 +556,6 @@ class FormRequestTest(RequestTest):
fs = _qs(req, to_unicode=True, encoding='latin1')
self.assertTrue(fs[u'price in \u00a5'])
-
def test_from_response_multiple_forms_clickdata(self):
response = _buildresponse(
"""<form name="form1">
@@ -989,7 +988,7 @@ class FormRequestTest(RequestTest):
"""
<html>
<head>
- <base href="http://b.com/">
+ <base href=" http://b.com/">
</head>
<body>
<form action="test_form">
@@ -1002,6 +1001,11 @@ class FormRequestTest(RequestTest):
req = self.request_class.from_response(response)
self.assertEqual(req.url, 'http://b.com/test_form')
+ def test_spaces_in_action(self):
+ resp = _buildresponse('<body><form action=" path\n"></form></body>')
+ req = self.request_class.from_response(resp)
+ self.assertEqual(req.url, 'http://example.com/path')
+
def test_from_response_css(self):
response = _buildresponse(
"""<form action="post.php" method="POST">
@@ -1023,12 +1027,14 @@ class FormRequestTest(RequestTest):
self.assertRaises(ValueError, self.request_class.from_response,
response, formcss="input[name='abc']")
+
def _buildresponse(body, **kwargs):
kwargs.setdefault('body', body)
kwargs.setdefault('url', 'http://example.com')
kwargs.setdefault('encoding', 'utf-8')
return HtmlResponse(**kwargs)
+
def _qs(req, encoding='utf-8', to_unicode=False):
if req.method == 'POST':
qs = req.body