Scrapy, a fast high-level web crawling & scraping framework for Python.
Update CI to support Twisted 21.2.0 (#5027)
Adrián Chaves308a58aa275a
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex aafd1d8b2..e3ab23ea5 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -9,9 +9,8 @@ from contextlib import suppressfrom io import BytesIOfrom itemadapter import ItemAdapter-from PIL import Image-from scrapy.exceptions import DropItem+from scrapy.exceptions import DropItem, NotConfiguredfrom scrapy.http import Requestfrom scrapy.pipelines.files import FileException, FilesPipeline# TODO: from scrapy.pipelines.media import MediaPipeline@@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline):DEFAULT_IMAGES_RESULT_FIELD = 'images'def __init__(self, store_uri, download_func=None, settings=None):+ try:+ from PIL import Image+ self._Image = Image+ except ImportError:+ raise NotConfigured(+ 'ImagesPipeline requires installing Pillow 4.0.0 or later'+ )+super().__init__(store_uri, settings=settings, download_func=download_func)if isinstance(settings, dict) or settings is None:@@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline):def get_images(self, response, request, info, *, item=None):path = self.file_path(request, response=response, info=info, item=item)- orig_image = Image.open(BytesIO(response.body))+ orig_image = self._Image.open(BytesIO(response.body))width, height = orig_image.sizeif width < self.min_width or height < self.min_height:@@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline):def convert_image(self, image, size=None):if image.format == 'PNG' and image.mode == 'RGBA':- background = Image.new('RGBA', image.size, (255, 255, 255))+ background = self._Image.new('RGBA', image.size, (255, 255, 255))background.paste(image, image)image = background.convert('RGB')elif image.mode == 'P':image = image.convert("RGBA")- background = Image.new('RGBA', image.size, (255, 255, 255))+ background = self._Image.new('RGBA', image.size, (255, 255, 255))background.paste(image, image)image = background.convert('RGB')elif image.mode != 'RGB':@@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline):if size:image = image.copy()- image.thumbnail(size, Image.ANTIALIAS)+ image.thumbnail(size, self._Image.ANTIALIAS)buf = BytesIO()image.save(buf, 'JPEG')diff --git a/tests/test_commands.py b/tests/test_commands.pyindex d3ac05eac..eec1f02ee 100644--- a/tests/test_commands.py+++ b/tests/test_commands.py@@ -17,6 +17,8 @@ from threading import Timerfrom unittest import skipIffrom pytest import mark+from twisted import version as twisted_version+from twisted.python.versions import Versionfrom twisted.trial import unittestimport scrapy@@ -630,6 +632,7 @@ class MySpider(scrapy.Spider):@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')def test_custom_asyncio_loop_enabled_true(self):log = self.get_log(self.debug_log_spider, args=['-s',diff --git a/tests/test_crawler.py b/tests/test_crawler.pyindex ab113710d..dec517bb6 100644--- a/tests/test_crawler.py+++ b/tests/test_crawler.py@@ -8,7 +8,9 @@ from unittest import skipIffrom pytest import raises, markfrom testfixtures import LogCapture+from twisted import version as twisted_versionfrom twisted.internet import defer+from twisted.python.versions import Versionfrom twisted.trial import unittestimport scrapy@@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):@mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')@mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')def test_custom_loop_asyncio(self):log = self.run_script("asyncio_custom_loop.py")self.assertIn("Spider closed (finished)", log)@@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):@mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")@mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")+ @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')def test_custom_loop_asyncio_deferred_signal(self):log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")self.assertIn("Spider closed (finished)", log)diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.pyindex 55fcfa7ba..f49fda701 100644--- a/tests/test_pipeline_crawl.py+++ b/tests/test_pipeline_crawl.py@@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase):self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)+try:+ from PIL import Image # noqa: imported just to check for the import error+except ImportError:+ skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'+else:+ skip_pillow = None++class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):++ skip = skip_pillow+pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'store_setting_key = 'IMAGES_STORE'media_key = 'images'diff --git a/tests/test_pipeline_images.py b/tests/test_pipeline_images.pyindex ad138a2dc..c69cd0e4a 100644--- a/tests/test_pipeline_images.py+++ b/tests/test_pipeline_images.py@@ -23,15 +23,16 @@ except ImportError:dataclass_field = None-skip = Falsetry:from PIL import Imageexcept ImportError:- skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'+ skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'else:encoders = {'jpeg_encoder', 'jpeg_decoder'}if not encoders.issubset(set(Image.core.__dict__)):- skip = 'Missing JPEG encoders'+ skip_pillow = 'Missing JPEG encoders'+ else:+ skip_pillow = Nonedef _mocked_download_func(request, info):@@ -41,7 +42,7 @@ def _mocked_download_func(request, info):class ImagesPipelineTestCase(unittest.TestCase):- skip = skip+ skip = skip_pillowdef setUp(self):self.tempdir = mkdtemp()@@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline):class ImagesPipelineTestCaseFieldsMixin:+ skip = skip_pillow+def test_item_fields_default(self):url = 'http://www.example.com/images/1.jpg'item = self.item_class(name='item1', image_urls=[url])@@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, uclass ImagesPipelineTestCaseCustomSettings(unittest.TestCase):++ skip = skip_pillow+img_cls_attribute_names = [# Pipeline attribute names with corresponding setting names.("EXPIRES", "IMAGES_EXPIRES"),diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.pyindex 6afd47497..893d43052 100644--- a/tests/test_pipeline_media.py+++ b/tests/test_pipeline_media.py@@ -1,3 +1,5 @@+from typing import Optional+from testfixtures import LogCapturefrom twisted.trial import unittestfrom twisted.python.failure import Failure@@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_allfrom scrapy import signals+try:+ from PIL import Image # noqa: imported just to check for the import error+except ImportError:+ skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'+else:+ skip_pillow = None++def _mocked_download_func(request, info):response = request.meta.get('response')return response() if callable(response) else response@@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):+ skip = skip_pillowdef setUp(self):self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)diff --git a/tox.ini b/tox.iniindex e0c69350e..6907c8906 100644--- a/tox.ini+++ b/tox.ini@@ -19,9 +19,6 @@ deps =mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'# Extrasbotocore>=1.4.87- Pillow>=4.0.0- # Twisted 21+ causes issues in tests that use skipIf- Twisted[http2]>=17.9.0,<21passenv =S3_TEST_FILE_URIAWS_ACCESS_KEY_ID@@ -124,6 +121,7 @@ deps ={[testenv]deps}reppyrobotexclusionrulesparser+ Pillow>=4.0.0[testenv:asyncio]commands =