Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Disallow media extensions unregistered with IANA (#3954)

OmarFarrag13735bcf34b0
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index ea06d2ae8..cc3d10b63 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -5,6 +5,7 @@ See documentation in topics/media-pipeline.rst
"""
import functools
import hashlib
+import mimetypes
import os
import os.path
import time
@@ -14,6 +15,7 @@ from six.moves.urllib.parse import urlparse
from collections import defaultdict
import six
+
try:
from cStringIO import StringIO as BytesIO
except ImportError:
@@ -473,4 +475,11 @@ class FilesPipeline(MediaPipeline):
def file_path(self, request, response=None, info=None):
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1]
+ # Handles empty and wild extensions by trying to guess the
+ # mime type then extension or default to empty string otherwise
+ if media_ext not in mimetypes.types_map:
+ media_ext = ''
+ media_type = mimetypes.guess_type(request.url)[0]
+ if media_type:
+ media_ext = mimetypes.guess_extension(media_type)
return 'full/%s%s' % (media_guid, media_ext)
diff --git a/tests/test_pipeline_files.py b/tests/test_pipeline_files.py
index 0c5aaaa44..cb8f8da18 100644
--- a/tests/test_pipeline_files.py
+++ b/tests/test_pipeline_files.py
@@ -57,6 +57,12 @@ class FilesPipelineTestCase(unittest.TestCase):
response=Response("http://www.dorma.co.uk/images/product_details/2532"),
info=object()),
'full/244e0dd7d96a3b7b01f54eded250c9e272577aa1')
+ self.assertEqual(file_path(Request("http://www.dfsonline.co.uk/get_prod_image.php?img=status_0907_mdm.jpg.bohaha")),
+ 'full/76c00cef2ef669ae65052661f68d451162829507')
+ self.assertEqual(file_path(Request("data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAR0AAACxCAMAAADOHZloAAACClBMVEX/\
+ //+F0tzCwMK76ZKQ21AMqr7oAAC96JvD5aWM2kvZ78J0N7fmAAC46Y4Ap7y")),
+ 'full/178059cbeba2e34120a67f2dc1afc3ecc09b61cb.png')
+
def test_fs_store(self):
assert isinstance(self.pipeline.store, FSFilesStore)