Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Update CI to support Twisted 21.2.0 (#5027)

Adrián Chaves308a58aa275a
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index aafd1d8b2..e3ab23ea5 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -9,9 +9,8 @@ from contextlib import suppress
from io import BytesIO
from itemadapter import ItemAdapter
-from PIL import Image
-from scrapy.exceptions import DropItem
+from scrapy.exceptions import DropItem, NotConfigured
from scrapy.http import Request
from scrapy.pipelines.files import FileException, FilesPipeline
# TODO: from scrapy.pipelines.media import MediaPipeline
@@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline):
DEFAULT_IMAGES_RESULT_FIELD = 'images'
def __init__(self, store_uri, download_func=None, settings=None):
+ try:
+ from PIL import Image
+ self._Image = Image
+ except ImportError:
+ raise NotConfigured(
+ 'ImagesPipeline requires installing Pillow 4.0.0 or later'
+ )
+
super().__init__(store_uri, settings=settings, download_func=download_func)
if isinstance(settings, dict) or settings is None:
@@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline):
def get_images(self, response, request, info, *, item=None):
path = self.file_path(request, response=response, info=info, item=item)
- orig_image = Image.open(BytesIO(response.body))
+ orig_image = self._Image.open(BytesIO(response.body))
width, height = orig_image.size
if width < self.min_width or height < self.min_height:
@@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline):
def convert_image(self, image, size=None):
if image.format == 'PNG' and image.mode == 'RGBA':
- background = Image.new('RGBA', image.size, (255, 255, 255))
+ background = self._Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
elif image.mode == 'P':
image = image.convert("RGBA")
- background = Image.new('RGBA', image.size, (255, 255, 255))
+ background = self._Image.new('RGBA', image.size, (255, 255, 255))
background.paste(image, image)
image = background.convert('RGB')
elif image.mode != 'RGB':
@@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline):
if size:
image = image.copy()
- image.thumbnail(size, Image.ANTIALIAS)
+ image.thumbnail(size, self._Image.ANTIALIAS)
buf = BytesIO()
image.save(buf, 'JPEG')
diff --git a/tests/test_commands.py b/tests/test_commands.py
index d3ac05eac..eec1f02ee 100644
--- a/tests/test_commands.py
+++ b/tests/test_commands.py
@@ -17,6 +17,8 @@ from threading import Timer
from unittest import skipIf
from pytest import mark
+from twisted import version as twisted_version
+from twisted.python.versions import Version
from twisted.trial import unittest
import scrapy
@@ -630,6 +632,7 @@ class MySpider(scrapy.Spider):
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
def test_custom_asyncio_loop_enabled_true(self):
log = self.get_log(self.debug_log_spider, args=[
'-s',
diff --git a/tests/test_crawler.py b/tests/test_crawler.py
index ab113710d..dec517bb6 100644
--- a/tests/test_crawler.py
+++ b/tests/test_crawler.py
@@ -8,7 +8,9 @@ from unittest import skipIf
from pytest import raises, mark
from testfixtures import LogCapture
+from twisted import version as twisted_version
from twisted.internet import defer
+from twisted.python.versions import Version
from twisted.trial import unittest
import scrapy
@@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
def test_custom_loop_asyncio(self):
log = self.run_script("asyncio_custom_loop.py")
self.assertIn("Spider closed (finished)", log)
@@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
@mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
@mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
def test_custom_loop_asyncio_deferred_signal(self):
log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
self.assertIn("Spider closed (finished)", log)
diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py
index 55fcfa7ba..f49fda701 100644
--- a/tests/test_pipeline_crawl.py
+++ b/tests/test_pipeline_crawl.py
@@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase):
self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
+try:
+ from PIL import Image # noqa: imported just to check for the import error
+except ImportError:
+ skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+ skip_pillow = None
+
+
class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
+
+ skip = skip_pillow
+
pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
store_setting_key = 'IMAGES_STORE'
media_key = 'images'
diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.py
index ad138a2dc..c69cd0e4a 100644
--- a/tests/test_pipeline_images.py
+++ b/tests/test_pipeline_images.py
@@ -23,15 +23,16 @@ except ImportError:
dataclass_field = None
-skip = False
try:
from PIL import Image
except ImportError:
- skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+ skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
else:
encoders = {'jpeg_encoder', 'jpeg_decoder'}
if not encoders.issubset(set(Image.core.__dict__)):
- skip = 'Missing JPEG encoders'
+ skip_pillow = 'Missing JPEG encoders'
+ else:
+ skip_pillow = None
def _mocked_download_func(request, info):
@@ -41,7 +42,7 @@ def _mocked_download_func(request, info):
class ImagesPipelineTestCase(unittest.TestCase):
- skip = skip
+ skip = skip_pillow
def setUp(self):
self.tempdir = mkdtemp()
@@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline):
class ImagesPipelineTestCaseFieldsMixin:
+ skip = skip_pillow
+
def test_item_fields_default(self):
url = 'http://www.example.com/images/1.jpg'
item = self.item_class(name='item1', image_urls=[url])
@@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u
class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+ skip = skip_pillow
+
img_cls_attribute_names = [
# Pipeline attribute names with corresponding setting names.
("EXPIRES", "IMAGES_EXPIRES"),
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 6afd47497..893d43052 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -1,3 +1,5 @@
+from typing import Optional
+
from testfixtures import LogCapture
from twisted.trial import unittest
from twisted.python.failure import Failure
@@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all
from scrapy import signals
+try:
+ from PIL import Image # noqa: imported just to check for the import error
+except ImportError:
+ skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+ skip_pillow = None
+
+
def _mocked_download_func(request, info):
response = request.meta.get('response')
return response() if callable(response) else response
@@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+ skip = skip_pillow
def setUp(self):
self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)
diff --git a/tox.ini b/tox.ini
index e0c69350e..6907c8906 100644
--- a/tox.ini
+++ b/tox.ini
@@ -19,9 +19,6 @@ deps =
mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
# Extras
botocore>=1.4.87
- Pillow>=4.0.0
- # Twisted 21+ causes issues in tests that use skipIf
- Twisted[http2]>=17.9.0,<21
passenv =
S3_TEST_FILE_URI
AWS_ACCESS_KEY_ID
@@ -124,6 +121,7 @@ deps =
{[testenv]deps}
reppy
robotexclusionrulesparser
+ Pillow>=4.0.0
[testenv:asyncio]
commands =