Scrapy, a fast high-level web crawling & scraping framework for Python.
More liberal Content-Disposition header parsing
Paul Tremberthcabed6f183cf
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.pyindex 4880cc7b9..c667b141d 100644--- a/scrapy/responsetypes.py+++ b/scrapy/responsetypes.py@@ -59,7 +59,8 @@ class ResponseTypes(object):def from_content_disposition(self, content_disposition):try:- filename = to_native_str(content_disposition).split(';')[1].split('=')[1]+ filename = to_native_str(content_disposition,+ encoding='latin-1', errors='replace').split(';')[1].split('=')[1]filename = filename.strip('"\'')return self.from_filename(filename)except IndexError:diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.pyindex 2374d518f..118136ac4 100644--- a/tests/test_responsetypes.py+++ b/tests/test_responsetypes.py@@ -1,3 +1,4 @@+# -*- coding: utf-8 -*-import unittestfrom scrapy.responsetypes import responsetypes@@ -20,8 +21,14 @@ class ResponseTypesTest(unittest.TestCase):def test_from_content_disposition(self):mappings = [- ('attachment; filename="data.xml"', XmlResponse),- ('attachment; filename=data.xml', XmlResponse),+ (b'attachment; filename="data.xml"', XmlResponse),+ (b'attachment; filename=data.xml', XmlResponse),+ (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),+ (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),+ (u'attachment;filename=data高.doc'.encode('gbk'), Response),+ (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),+ (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),+]for source, cls in mappings:retcls = responsetypes.from_content_disposition(source)