Scrapy, a fast high-level web crawling & scraping framework for Python.
Use six.PY2 instead of six.PY3 for Python version variations
Paul Tremberth25401fd322b6
diff --git a/scrapy/utils/url.py b/scrapy/utils/url.pyindex 716513648..069666dbc 100644--- a/scrapy/utils/url.py+++ b/scrapy/utils/url.py@@ -83,17 +83,16 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,try:scheme, netloc, path, params, query, fragment = _safe_ParseResult(parse_url(url), encoding=encoding)- except UnicodeError as e:- if encoding != 'utf8':- scheme, netloc, path, params, query, fragment = _safe_ParseResult(- parse_url(url), encoding='utf8')- else:- raise+ except UnicodeEncodeError as e:+ scheme, netloc, path, params, query, fragment = _safe_ParseResult(+ parse_url(url), encoding='utf8')# 1. decode query-string as UTF-8 (or keep raw bytes),# sort values,# and percent-encode them back- if not six.PY2:+ if six.PY2:+ keyvals = parse_qsl(query, keep_blank_values)+ else:# Python3's urllib.parse.parse_qsl does not work as wanted# for percent-encoded characters that do not match passed encoding,# they get lost.@@ -118,8 +117,6 @@ def canonicalize_url(url, keep_blank_values=True, keep_fragments=False,# IRIs (namely, to be able to include non-ASCII characters) can only be# used if the query part is encoded in UTF-8.keyvals = parse_qsl_to_bytes(query, keep_blank_values)- else:- keyvals = parse_qsl(query, keep_blank_values)keyvals.sort()query = urlencode(keyvals)@@ -138,16 +135,17 @@ def _unquotepath(path):for reserved in ('2f', '2F', '3f', '3F'):path = path.replace('%' + reserved, '%25' + reserved.upper())- if six.PY3:- # standard lib's unquote() does not work in Python 3- # for non-UTF-8 percent-escaped characters, they get lost.+ if six.PY2:+ # in Python 2, '%a3' becomes '\xa3', which is what we want+ return unquote(path)+ else:+ # in Python 3,+ # standard lib's unquote() does not work for non-UTF-8+ # percent-escaped characters, they get lost.# e.g., '%a3' becomes 'REPLACEMENT CHARACTER' (U+FFFD)## unquote_to_bytes() returns raw bytes insteadreturn unquote_to_bytes(path)- else:- # in Python 2, '%a3' becomes '\xa3', which is what we want- return unquote(path)def parse_url(url, encoding=None):