Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

data URI download handler.

Artur Gaspard60642e1755b
diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.py
new file mode 100644
index 000000000..637a6f9c0
--- /dev/null
+++ b/scrapy/core/downloader/handlers/data.py
@@ -0,0 +1,93 @@
+import base64
+import re
+from six.moves.urllib.parse import unquote
+
+from scrapy.http import TextResponse
+from scrapy.responsetypes import responsetypes
+from scrapy.utils.datatypes import CaselessDict
+from scrapy.utils.decorators import defers
+
+
+# ASCII characters.
+_char = set(map(chr, range(127)))
+
+# RFC 2045 token.
+_token = r'[{}]+'.format(re.escape(''.join(_char -
+ # Control characters.
+ set(map(chr, range(0, 32))) -
+ # tspecials and space.
+ set('()<>@,;:\\"/[]?= '))))
+
+# RFC 822 quoted-string, without surrounding quotation marks.
+_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(
+ re.escape(''.join(_char - {'"', '\\', '\r'})),
+ re.escape(''.join(_char))
+)
+
+# RFC 2397 mediatype.
+_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))
+
+_mediatype_parameter_pattern = re.compile(
+ r';({token})=(?:({token})|"({quoted})")'.format(token=_token,
+ quoted=_quoted_string)
+)
+
+
+class DataURIDownloadHandler(object):
+ def __init__(self, settings):
+ super(DataURIDownloadHandler, self).__init__()
+
+ @defers
+ def download_request(self, request, spider):
+ url = request.url
+
+ scheme, url = url.split(':', 1)
+ if scheme != 'data':
+ raise ValueError("not a data URI")
+
+ # RFC 3986 section 2.1 allows percent encoding to escape characters
+ # that would be interpreted as delimiters, implying that actual
+ # delimiters should not be percent-encoded.
+ # Decoding before parsing will allow malformed URIs with
+ # percent-encoded delimiters, but it makes parsing easier and should
+ # not affect well-formed URIs, as the delimiters used in this URI
+ # scheme are not allowed, percent-encoded or not, in tokens.
+ url = unquote(url)
+
+ media_type = "text/plain"
+ media_type_params = CaselessDict()
+
+ m = _mediatype_pattern.match(url)
+ if m:
+ media_type = m.group()
+ url = url[m.end():]
+ else:
+ media_type_params['charset'] = "US-ASCII"
+
+ while True:
+ m = _mediatype_parameter_pattern.match(url)
+ if m:
+ attribute, value, value_quoted = m.groups()
+ if value_quoted:
+ value = re.sub(r'\\(.)', '\1', value_quoted)
+ media_type_params[attribute] = value
+ url = url[m.end():]
+ else:
+ break
+
+ is_base64, data = url.split(',', 1)
+ if is_base64:
+ if is_base64 != ";base64":
+ raise ValueError("invalid data URI")
+ data = base64.b64decode(data)
+
+ respcls = responsetypes.from_mimetype(media_type)
+
+ resp_kwargs = {}
+
+ if media_type:
+ media_type = media_type.split('/')
+ if issubclass(respcls, TextResponse) and media_type[0] == 'text':
+ resp_kwargs['encoding'] = media_type_params.get('charset')
+
+ return respcls(url=request.url, body=data, **resp_kwargs)
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index c1683fb3e..c21a1670f 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -6,7 +6,6 @@ try:
from unittest import mock
except ImportError:
import mock
-import shutil
from twisted.trial import unittest
from twisted.protocols.policies import WrappingFactory
@@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentials
from w3lib.url import path_to_file_uri
from scrapy.core.downloader.handlers import DownloadHandlers
+from scrapy.core.downloader.handlers.data import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
@@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
from scrapy.spiders import Spider
from scrapy.http import Request
from scrapy.http.response.text import TextResponse
+from scrapy.responsetypes import responsetypes
from scrapy.settings import Settings
from scrapy.utils.test import get_crawler, skip_if_no_boto
from scrapy.utils.python import to_bytes
@@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase):
def tearDown(self):
shutil.rmtree(self.directory)
+
+
+class DataURITestCase(unittest.TestCase):
+
+ def setUp(self):
+ self.download_handler = DataURIDownloadHandler(Settings())
+ self.download_request = self.download_handler.download_request
+ self.spider = Spider('foo')
+
+ def test_default_mediatype_encoding(self):
+ def _test(response):
+ self.assertEquals(response.text, 'A brief note')
+ self.assertEquals(type(response),
+ responsetypes.from_mimetype("text/plain"))
+ self.assertEquals(response.encoding, "US-ASCII")
+
+ request = Request("data:,A%20brief%20note")
+ return self.download_request(request, self.spider).addCallback(_test)
+
+ def test_default_mediatype(self):
+ def _test(response):
+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+ self.assertEquals(type(response),
+ responsetypes.from_mimetype("text/plain"))
+ self.assertEquals(response.encoding, "iso-8859-7")
+
+ request = Request("data:;charset=iso-8859-7,%be%d3%be")
+ return self.download_request(request, self.spider).addCallback(_test)
+
+ def test_text_charset(self):
+ def _test(response):
+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+ self.assertEquals(response.encoding, "iso-8859-7")
+
+ request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")
+ return self.download_request(request, self.spider).addCallback(_test)
+
+ def test_mediatype_parameters(self):
+ def _test(response):
+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')
+ self.assertEquals(type(response),
+ responsetypes.from_mimetype("text/plain"))
+ self.assertEquals(response.encoding, "utf-8")
+
+ request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'
+ 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'
+ ',%CE%8E%CE%A3%CE%8E')
+ return self.download_request(request, self.spider).addCallback(_test)
+
+ def test_base64(self):
+ def _test(response):
+ self.assertEquals(response.text, 'Hello, world.')
+
+ request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')
+ return self.download_request(request, self.spider).addCallback(_test)