Scrapy, a fast high-level web crawling & scraping framework for Python.
Do not interpret non-ASCII bytes in "Location" and percent-encode them (#2322)
Paul Tremberthdb408528928b
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.pyindex 4ed7e4c24..db276eefb 100644--- a/scrapy/downloadermiddlewares/redirect.py+++ b/scrapy/downloadermiddlewares/redirect.py@@ -1,9 +1,10 @@import loggingfrom six.moves.urllib.parse import urljoin+from w3lib.url import safe_url_string+from scrapy.http import HtmlResponsefrom scrapy.utils.response import get_meta_refresh-from scrapy.utils.python import to_native_strfrom scrapy.exceptions import IgnoreRequest, NotConfiguredlogger = logging.getLogger(__name__)@@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):if 'Location' not in response.headers or response.status not in allowed_status:return response- # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8- location = to_native_str(response.headers['location'].decode('latin1'))+ location = safe_url_string(response.headers['location'])redirected_url = urljoin(request.url, location)diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.pyindex 9db073cc5..e8c92affa 100644--- a/tests/test_downloadermiddleware_redirect.py+++ b/tests/test_downloadermiddleware_redirect.py@@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase):latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)req_result = self.mw.process_response(req, resp, self.spider)- perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'+ perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'self.assertEquals(perc_encoded_utf8_url, req_result.url)- def test_location_with_wrong_encoding(self):+ def test_utf8_location(self):req = Request('http://scrapytest.org/first')- utf8_location = u'/ação' # header with wrong encoding (utf-8)+ utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encodingresp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)req_result = self.mw.process_response(req, resp, self.spider)- perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'+ perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'self.assertEquals(perc_encoded_utf8_url, req_result.url)