Scrapy, a fast high-level web crawling & scraping framework for Python.
Ensure bytes objects when needed in data URI downloader.
Artur Gaspar7e9f2c31d78d
diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.pyindex 637a6f9c0..30c39865f 100644--- a/scrapy/core/downloader/handlers/data.py+++ b/scrapy/core/downloader/handlers/data.py@@ -1,6 +1,12 @@import base64import re-from six.moves.urllib.parse import unquote++import six++if six.PY2:+ from urllib import unquote+else:+ from urllib.parse import unquote_to_bytes as unquotefrom scrapy.http import TextResponsefrom scrapy.responsetypes import responsetypes@@ -24,12 +30,19 @@ _quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(re.escape(''.join(_char)))+# Encode the regular expression strings to make them into bytes, as Python 3+# bytes have no format() method, but bytes must be passed to re.compile() in+# order to make a pattern object that can be used to match on bytes.+# RFC 2397 mediatype.-_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))+_mediatype_pattern = re.compile(+ r'{token}/{token}'.format(token=_token).encode()+)_mediatype_parameter_pattern = re.compile(r';({token})=(?:({token})|"({quoted})")'.format(token=_token,- quoted=_quoted_string)+ quoted=_quoted_string+ ).encode())@@ -59,7 +72,7 @@ class DataURIDownloadHandler(object):m = _mediatype_pattern.match(url)if m:- media_type = m.group()+ media_type = m.group().decode()url = url[m.end():]else:media_type_params['charset'] = "US-ASCII"@@ -69,15 +82,15 @@ class DataURIDownloadHandler(object):if m:attribute, value, value_quoted = m.groups()if value_quoted:- value = re.sub(r'\\(.)', '\1', value_quoted)- media_type_params[attribute] = value+ value = re.sub(br'\\(.)', r'\1', value_quoted)+ media_type_params[attribute.decode()] = value.decode()url = url[m.end():]else:break- is_base64, data = url.split(',', 1)+ is_base64, data = url.split(b',', 1)if is_base64:- if is_base64 != ";base64":+ if is_base64 != b";base64":raise ValueError("invalid data URI")data = base64.b64decode(data)