Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Do not interpret non-ASCII bytes in "Location" and percent-encode them (#2322)

Paul Tremberthdb408528928b
diff --git a/scrapy/downloadermiddlewares/redirect.py b/scrapy/downloadermiddlewares/redirect.py
index 4ed7e4c24..db276eefb 100644
--- a/scrapy/downloadermiddlewares/redirect.py
+++ b/scrapy/downloadermiddlewares/redirect.py
@@ -1,9 +1,10 @@
import logging
from six.moves.urllib.parse import urljoin
+from w3lib.url import safe_url_string
+
from scrapy.http import HtmlResponse
from scrapy.utils.response import get_meta_refresh
-from scrapy.utils.python import to_native_str
from scrapy.exceptions import IgnoreRequest, NotConfigured
logger = logging.getLogger(__name__)
@@ -65,8 +66,7 @@ class RedirectMiddleware(BaseRedirectMiddleware):
if 'Location' not in response.headers or response.status not in allowed_status:
return response
- # HTTP header is ascii or latin1, redirected url will be percent-encoded utf-8
- location = to_native_str(response.headers['location'].decode('latin1'))
+ location = safe_url_string(response.headers['location'])
redirected_url = urljoin(request.url, location)
diff --git a/tests/test_downloadermiddleware_redirect.py b/tests/test_downloadermiddleware_redirect.py
index 9db073cc5..e8c92affa 100644
--- a/tests/test_downloadermiddleware_redirect.py
+++ b/tests/test_downloadermiddleware_redirect.py
@@ -157,15 +157,15 @@ class RedirectMiddlewareTest(unittest.TestCase):
latin1_location = u'/ação'.encode('latin1') # HTTP historically supports latin1
resp = Response('http://scrapytest.org/first', headers={'Location': latin1_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
- perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
+ perc_encoded_utf8_url = 'http://scrapytest.org/a%E7%E3o'
self.assertEquals(perc_encoded_utf8_url, req_result.url)
- def test_location_with_wrong_encoding(self):
+ def test_utf8_location(self):
req = Request('http://scrapytest.org/first')
- utf8_location = u'/ação' # header with wrong encoding (utf-8)
+ utf8_location = u'/ação'.encode('utf-8') # header using UTF-8 encoding
resp = Response('http://scrapytest.org/first', headers={'Location': utf8_location}, status=302)
req_result = self.mw.process_response(req, resp, self.spider)
- perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%83%C2%A7%C3%83%C2%A3o'
+ perc_encoded_utf8_url = 'http://scrapytest.org/a%C3%A7%C3%A3o'
self.assertEquals(perc_encoded_utf8_url, req_result.url)