Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Use w3lib data URI parser.

Artur Gasparc847e7d4d006
diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py
index 30c39865f..d102f2b73 100644
--- a/scrapy/core/downloader/handlers/data.py
+++ b/scrapy/core/downloader/handlers/data.py
@@ -1,106 +1,23 @@
-import base64
-import re
-
-import six
-
-if six.PY2:
- from urllib import unquote
-else:
- from urllib.parse import unquote_to_bytes as unquote
+from w3lib.url import parse_data_uri
from scrapy.http import TextResponse
from scrapy.responsetypes import responsetypes
-from scrapy.utils.datatypes import CaselessDict
from scrapy.utils.decorators import defers
-# ASCII characters.
-_char = set(map(chr, range(127)))
-
-# RFC 2045 token.
-_token = r'[{}]+'.format(re.escape(''.join(_char -
- # Control characters.
- set(map(chr, range(0, 32))) -
- # tspecials and space.
- set('()<>@,;:\\"/[]?= '))))
-
-# RFC 822 quoted-string, without surrounding quotation marks.
-_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
- re.escape(''.join(_char - {'"', '\\', '\r'})),
- re.escape(''.join(_char))
-)
-
-# Encode the regular expression strings to make them into bytes, as Python 3
-# bytes have no format() method, but bytes must be passed to re.compile() in
-# order to make a pattern object that can be used to match on bytes.
-
-# RFC 2397 mediatype.
-_mediatype_pattern = re.compile(
- r'{token}/{token}'.format(token=_token).encode()
-)
-
-_mediatype_parameter_pattern = re.compile(
- r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
- quoted=_quoted_string
- ).encode()
-)
-
-
class DataURIDownloadHandler(object):
def __init__(self, settings):
super(DataURIDownloadHandler, self).__init__()
@defers
def download_request(self, request, spider):
- url = request.url
-
- scheme, url = url.split(':', 1)
- if scheme != 'data':
- raise ValueError("not a data URI")
-
- # RFC 3986 section 2.1 allows percent encoding to escape characters
- # that would be interpreted as delimiters, implying that actual
- # delimiters should not be percent-encoded.
- # Decoding before parsing will allow malformed URIs with
- # percent-encoded delimiters, but it makes parsing easier and should
- # not affect well-formed URIs, as the delimiters used in this URI
- # scheme are not allowed, percent-encoded or not, in tokens.
- url = unquote(url)
-
- media_type = "text/plain"
- media_type_params = CaselessDict()
-
- m = _mediatype_pattern.match(url)
- if m:
- media_type = m.group().decode()
- url = url[m.end():]
- else:
- media_type_params['charset'] = "US-ASCII"
-
- while True:
- m = _mediatype_parameter_pattern.match(url)
- if m:
- attribute, value, value_quoted = m.groups()
- if value_quoted:
- value = re.sub(br'\\(.)', r'\1', value_quoted)
- media_type_params[attribute.decode()] = value.decode()
- url = url[m.end():]
- else:
- break
-
- is_base64, data = url.split(b',', 1)
- if is_base64:
- if is_base64 != b";base64":
- raise ValueError("invalid data URI")
- data = base64.b64decode(data)
-
- respcls = responsetypes.from_mimetype(media_type)
+ uri = parse_data_uri(request.url)
+ respcls = responsetypes.from_mimetype(uri.media_type)
resp_kwargs = {}
+ if (issubclass(respcls, TextResponse) and
+ uri.media_type.split('/')[0] == 'text'):
+ charset = uri.media_type_parameters.get('charset')
+ resp_kwargs['encoding'] = charset
- if media_type:
- media_type = media_type.split('/')
- if issubclass(respcls, TextResponse) and media_type[0] == 'text':
- resp_kwargs['encoding'] = media_type_params.get('charset')
-
- return respcls(url=request.url, body=data, **resp_kwargs)
+ return respcls(url=request.url, body=uri.data, **resp_kwargs)