Scrapy, a fast high-level web crawling & scraping framework for Python.
data URI download handler.
Artur Gaspard60642e1755b
diff --git a/scrapy/core/downloader/handlers/data.py b/scrapy/core/downloader/handlers/data.pynew file mode 100644index 000000000..637a6f9c0--- /dev/null+++ b/scrapy/core/downloader/handlers/data.py@@ -0,0 +1,93 @@+import base64+import re+from six.moves.urllib.parse import unquote++from scrapy.http import TextResponse+from scrapy.responsetypes import responsetypes+from scrapy.utils.datatypes import CaselessDict+from scrapy.utils.decorators import defers+++# ASCII characters.+_char = set(map(chr, range(127)))++# RFC 2045 token.+_token = r'[{}]+'.format(re.escape(''.join(_char -+ # Control characters.+ set(map(chr, range(0, 32))) -+ # tspecials and space.+ set('()<>@,;:\\"/[]?= '))))++# RFC 822 quoted-string, without surrounding quotation marks.+_quoted_string = r'(?:[{}]|(?:\\[{}]))*'.format(+ re.escape(''.join(_char - {'"', '\\', '\r'})),+ re.escape(''.join(_char))+)++# RFC 2397 mediatype.+_mediatype_pattern = re.compile(r'{token}/{token}'.format(token=_token))++_mediatype_parameter_pattern = re.compile(+ r';({token})=(?:({token})|"({quoted})")'.format(token=_token,+ quoted=_quoted_string)+)+++class DataURIDownloadHandler(object):+ def __init__(self, settings):+ super(DataURIDownloadHandler, self).__init__()++ @defers+ def download_request(self, request, spider):+ url = request.url++ scheme, url = url.split(':', 1)+ if scheme != 'data':+ raise ValueError("not a data URI")++ # RFC 3986 section 2.1 allows percent encoding to escape characters+ # that would be interpreted as delimiters, implying that actual+ # delimiters should not be percent-encoded.+ # Decoding before parsing will allow malformed URIs with+ # percent-encoded delimiters, but it makes parsing easier and should+ # not affect well-formed URIs, as the delimiters used in this URI+ # scheme are not allowed, percent-encoded or not, in tokens.+ url = unquote(url)++ media_type = "text/plain"+ media_type_params = CaselessDict()++ m = _mediatype_pattern.match(url)+ if m:+ media_type = m.group()+ url = url[m.end():]+ else:+ media_type_params['charset'] = "US-ASCII"++ while True:+ m = _mediatype_parameter_pattern.match(url)+ if m:+ attribute, value, value_quoted = m.groups()+ if value_quoted:+ value = re.sub(r'\\(.)', '\1', value_quoted)+ media_type_params[attribute] = value+ url = url[m.end():]+ else:+ break++ is_base64, data = url.split(',', 1)+ if is_base64:+ if is_base64 != ";base64":+ raise ValueError("invalid data URI")+ data = base64.b64decode(data)++ respcls = responsetypes.from_mimetype(media_type)++ resp_kwargs = {}++ if media_type:+ media_type = media_type.split('/')+ if issubclass(respcls, TextResponse) and media_type[0] == 'text':+ resp_kwargs['encoding'] = media_type_params.get('charset')++ return respcls(url=request.url, body=data, **resp_kwargs)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex c1683fb3e..c21a1670f 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -6,7 +6,6 @@ try:from unittest import mockexcept ImportError:import mock-import shutilfrom twisted.trial import unittestfrom twisted.protocols.policies import WrappingFactory@@ -20,6 +19,7 @@ from twisted.cred import portal, checkers, credentialsfrom w3lib.url import path_to_file_urifrom scrapy.core.downloader.handlers import DownloadHandlers+from scrapy.core.downloader.handlers.data import DataURIDownloadHandlerfrom scrapy.core.downloader.handlers.file import FileDownloadHandlerfrom scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandlerfrom scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler@@ -29,6 +29,7 @@ from scrapy.core.downloader.handlers.s3 import S3DownloadHandlerfrom scrapy.spiders import Spiderfrom scrapy.http import Requestfrom scrapy.http.response.text import TextResponse+from scrapy.responsetypes import responsetypesfrom scrapy.settings import Settingsfrom scrapy.utils.test import get_crawler, skip_if_no_botofrom scrapy.utils.python import to_bytes@@ -828,3 +829,58 @@ class AnonymousFTPTestCase(BaseFTPTestCase):def tearDown(self):shutil.rmtree(self.directory)+++class DataURITestCase(unittest.TestCase):++ def setUp(self):+ self.download_handler = DataURIDownloadHandler(Settings())+ self.download_request = self.download_handler.download_request+ self.spider = Spider('foo')++ def test_default_mediatype_encoding(self):+ def _test(response):+ self.assertEquals(response.text, 'A brief note')+ self.assertEquals(type(response),+ responsetypes.from_mimetype("text/plain"))+ self.assertEquals(response.encoding, "US-ASCII")++ request = Request("data:,A%20brief%20note")+ return self.download_request(request, self.spider).addCallback(_test)++ def test_default_mediatype(self):+ def _test(response):+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')+ self.assertEquals(type(response),+ responsetypes.from_mimetype("text/plain"))+ self.assertEquals(response.encoding, "iso-8859-7")++ request = Request("data:;charset=iso-8859-7,%be%d3%be")+ return self.download_request(request, self.spider).addCallback(_test)++ def test_text_charset(self):+ def _test(response):+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')+ self.assertEquals(response.encoding, "iso-8859-7")++ request = Request("data:text/plain;charset=iso-8859-7,%be%d3%be")+ return self.download_request(request, self.spider).addCallback(_test)++ def test_mediatype_parameters(self):+ def _test(response):+ self.assertEquals(response.text, u'\u038e\u03a3\u038e')+ self.assertEquals(type(response),+ responsetypes.from_mimetype("text/plain"))+ self.assertEquals(response.encoding, "utf-8")++ request = Request('data:text/plain;foo=%22foo;bar%5C%22%22;'+ 'charset=utf-8;bar=%22foo;%5C%22 foo ;/,%22'+ ',%CE%8E%CE%A3%CE%8E')+ return self.download_request(request, self.spider).addCallback(_test)++ def test_base64(self):+ def _test(response):+ self.assertEquals(response.text, 'Hello, world.')++ request = Request('data:text/plain;base64,SGVsbG8sIHdvcmxkLg%3D%3D')+ return self.download_request(request, self.spider).addCallback(_test)