Scrapy, a fast high-level web crawling & scraping framework for Python.
get_media_requests: support and encourage callback=NO_CALLBACK
Adrián Chaves389fd99e7937
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.pyindex 01a9c41fe..91fc172b2 100644--- a/scrapy/pipelines/files.py+++ b/scrapy/pipelines/files.py@@ -22,6 +22,7 @@ from twisted.internet import defer, threadsfrom scrapy.exceptions import IgnoreRequest, NotConfiguredfrom scrapy.http import Request+from scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.media import MediaPipelinefrom scrapy.settings import Settingsfrom scrapy.utils.boto import is_botocore_available@@ -517,7 +518,7 @@ class FilesPipeline(MediaPipeline):# Overridable Interfacedef get_media_requests(self, item, info):urls = ItemAdapter(item).get(self.files_urls_field, [])- return [Request(u) for u in urls]+ return [Request(u, callback=NO_CALLBACK) for u in urls]def file_downloaded(self, response, request, info, *, item=None):path = self.file_path(request, response=response, info=info, item=item)diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.pyindex 0cfa5665a..9d18144ee 100644--- a/scrapy/pipelines/images.py+++ b/scrapy/pipelines/images.py@@ -13,6 +13,7 @@ from itemadapter import ItemAdapterfrom scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarningfrom scrapy.http import Request+from scrapy.http.request import NO_CALLBACKfrom scrapy.pipelines.files import FileException, FilesPipeline# TODO: from scrapy.pipelines.media import MediaPipeline@@ -214,7 +215,7 @@ class ImagesPipeline(FilesPipeline):def get_media_requests(self, item, info):urls = ItemAdapter(item).get(self.images_urls_field, [])- return [Request(u) for u in urls]+ return [Request(u, callback=NO_CALLBACK) for u in urls]def item_completed(self, results, item, info):with suppress(KeyError):diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.pyindex 1e921f0b5..679035c5d 100644--- a/scrapy/pipelines/media.py+++ b/scrapy/pipelines/media.py@@ -18,6 +18,10 @@ from scrapy.utils.log import failure_to_exc_infologger = logging.getLogger(__name__)+def _DUMMY_CALLBACK(response):+ return response++class MediaPipeline:LOG_FAILED_RESULTS = True@@ -91,7 +95,10 @@ class MediaPipeline:def _process_request(self, request, info, item):fp = self._fingerprinter.fingerprint(request)- cb = request.callback or (lambda _: _)+ if not request.callback or request.callback is NO_CALLBACK:+ cb = _DUMMY_CALLBACK+ else:+ cb = request.callbackeb = request.errbackrequest.callback = NO_CALLBACKrequest.errback = None