Scrapy, a fast high-level web crawling & scraping framework for Python.
Disallow media extensions unregistered with IANA (#3954)
OmarFarrag13735bcf34b0
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex ea06d2ae8..cc3d10b63 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst"""import functoolsimport hashlib+import mimetypesimport osimport os.pathimport time@@ -14,6 +15,7 @@ from six.moves.urllib.parse import urlparsefrom collections import defaultdictimport six+try:from cStringIO import StringIO as BytesIOexcept ImportError:@@ -473,4 +475,11 @@ class FilesPipeline(MediaPipeline):def file_path(self, request, response=None, info=None):media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()media_ext = os.path.splitext(request.url)[1]+ # Handles empty and wild extensions by trying to guess the+ # mime type then extension or default to empty string otherwise+ if media_ext not in mimetypes.types_map:+ media_ext = ''+ media_type = mimetypes.guess_type(request.url)[0]+ if media_type:+ media_ext = mimetypes.guess_extension(media_type)return 'full/%s%s' % (media_guid, media_ext)diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.pyindex 0c5aaaa44..cb8f8da18 100644--- a/tests/test_pipeline_files.py+++ b/tests/test_pipeline_files.py@@ -57,6 +57,12 @@ class FilesPipelineTestCase(unittest.TestCase):response=Response("http://www.dorma.co.uk/images/product_details/2532"),info=object()),'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')+ self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),+ 'full/76c00cef2ef669ae65052661f68d451162829507')+ self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\+ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),+ 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')+def test_fs_store(self):assert isinstance(self.pipeline.store, FSFilesStore)