Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

More liberal Content-Disposition header parsing

Paul Tremberthcabed6f183cf
diff --git a/scrapy/responsetypes.py b/scrapy/responsetypes.py
index 4880cc7b9..c667b141d 100644
--- a/scrapy/responsetypes.py
+++ b/scrapy/responsetypes.py
@@ -59,7 +59,8 @@ class ResponseTypes(object):
def from_content_disposition(self, content_disposition):
try:
- filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
+ filename = to_native_str(content_disposition,
+ encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
filename = filename.strip('"\'')
return self.from_filename(filename)
except IndexError:
diff --git a/tests/test_responsetypes.py b/tests/test_responsetypes.py
index 2374d518f..118136ac4 100644
--- a/tests/test_responsetypes.py
+++ b/tests/test_responsetypes.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
import unittest
from scrapy.responsetypes import responsetypes
@@ -20,8 +21,14 @@ class ResponseTypesTest(unittest.TestCase):
def test_from_content_disposition(self):
mappings = [
- ('attachment; filename="data.xml"', XmlResponse),
- ('attachment; filename=data.xml', XmlResponse),
+ (b'attachment; filename="data.xml"', XmlResponse),
+ (b'attachment; filename=data.xml', XmlResponse),
+ (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+ (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+ (u'attachment;filename=data高.doc'.encode('gbk'), Response),
+ (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+ (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+
]
for source, cls in mappings:
retcls = responsetypes.from_content_disposition(source)