Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

add tests

Mohammadtaher Abbasi5c586d78f0e1
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index c69cd0e4a..dd94d296b 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -93,6 +93,22 @@ class ImagesPipelineTestCase(unittest.TestCase):
info=object()),
'thumbs/50/850233df65a5b83361798f532f1fc549cd13cbe9.jpg')
+ def test_thumbnail_name_from_item(self):
+ """
+ Custom thumbnail name based on item data, overriding default implementation
+ """
+
+ class CustomImagesPipeline(ImagesPipeline):
+ def thumb_path(self, request, thumb_id, response=None, info=None, item=None):
+ return f"thumb/{thumb_id}/{item.get('path')}"
+
+ thumb_path = CustomImagesPipeline.from_settings(Settings(
+ {'IMAGES_STORE': self.tempdir}
+ )).thumb_path
+ item = dict(path='path-to-store-file')
+ request = Request("http://example.com")
+ self.assertEqual(thumb_path(request, 'small', item=item), 'thumb/small/path-to-store-file')
+
def test_convert_image(self):
SIZE = (100, 100)
# straigh forward case: RGB and JPEG
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 893d43052..a802c7cf1 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,4 +1,5 @@
from typing import Optional
+import io
from testfixtures import LogCapture
from twisted.trial import unittest
@@ -355,9 +356,12 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
def get_media_requests(self, item, info):
item_url = item['image_urls'][0]
+ output_img = io.BytesIO()
+ img = Image.new('RGB', (60, 30), color='red')
+ img.save(output_img, format='JPEG')
return Request(
item_url,
- meta={'response': Response(item_url, status=200, body=b'data')}
+ meta={'response': Response(item_url, status=200, body=output_img.getvalue())}
)
def inc_stats(self, *args, **kwargs):
@@ -379,9 +383,13 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
self._mockcalled.append('file_path')
return super(MockedMediaPipelineDeprecatedMethods, self).file_path(request, response, info)
+ def thumb_path(self, request, thumb_id, response=None, info=None):
+ self._mockcalled.append('thumb_path')
+ return super(MockedMediaPipelineDeprecatedMethods, self).thumb_path(request, thumb_id, response, info)
+
def get_images(self, response, request, info):
self._mockcalled.append('get_images')
- return []
+ return super(MockedMediaPipelineDeprecatedMethods, self).get_images(response, request, info)
def image_downloaded(self, response, request, info):
self._mockcalled.append('image_downloaded')
@@ -392,7 +400,11 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
skip = skip_pillow
def setUp(self):
- self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
+ self.pipe = MockedMediaPipelineDeprecatedMethods(
+ store_uri='store-uri',
+ download_func=_mocked_download_func,
+ settings=Settings({"IMAGES_THUMBS": {'small': (50, 50)}})
+ )
self.pipe.open_spider(None)
self.item = dict(image_urls=['http://picsum.photos/id/1014/200/300'], images=[])
@@ -444,6 +456,16 @@ class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
)
self._assert_method_called_with_warnings('file_path', message, warnings)
+ @inlineCallbacks
+ def test_thumb_path_called(self):
+ yield self.pipe.process_item(self.item, None)
+ warnings = self.flushWarnings([MediaPipeline._compatible])
+ message = (
+ 'thumb_path(self, request, thumb_id, response=None, info=None) is deprecated, '
+ 'please use thumb_path(self, request, thumb_id, response=None, info=None, *, item=None)'
+ )
+ self._assert_method_called_with_warnings('thumb_path', message, warnings)
+
@inlineCallbacks
def test_get_images_called(self):
yield self.pipe.process_item(self.item, None)