Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

get_media_requests: support and encourage callback=NO_CALLBACK

Adrián Chaves389fd99e7937
diff --git a/scrapy/pipelines/files.py b/scrapy/pipelines/files.py
index 01a9c41fe..91fc172b2 100644
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@@ -22,6 +22,7 @@ from twisted.internet import defer, threads
from scrapy.exceptions import IgnoreRequest, NotConfigured
from scrapy.http import Request
+from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.media import MediaPipeline
from scrapy.settings import Settings
from scrapy.utils.boto import is_botocore_available
@@ -517,7 +518,7 @@ class FilesPipeline(MediaPipeline):
# Overridable Interface
def get_media_requests(self, item, info):
urls = ItemAdapter(item).get(self.files_urls_field, [])
- return [Request(u) for u in urls]
+ return [Request(u, callback=NO_CALLBACK) for u in urls]
def file_downloaded(self, response, request, info, *, item=None):
path = self.file_path(request, response=response, info=info, item=item)
diff --git a/scrapy/pipelines/images.py b/scrapy/pipelines/images.py
index 0cfa5665a..9d18144ee 100644
--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@@ -13,6 +13,7 @@ from itemadapter import ItemAdapter
from scrapy.exceptions import DropItem, NotConfigured, ScrapyDeprecationWarning
from scrapy.http import Request
+from scrapy.http.request import NO_CALLBACK
from scrapy.pipelines.files import FileException, FilesPipeline
# TODO: from scrapy.pipelines.media import MediaPipeline
@@ -214,7 +215,7 @@ class ImagesPipeline(FilesPipeline):
def get_media_requests(self, item, info):
urls = ItemAdapter(item).get(self.images_urls_field, [])
- return [Request(u) for u in urls]
+ return [Request(u, callback=NO_CALLBACK) for u in urls]
def item_completed(self, results, item, info):
with suppress(KeyError):
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 1e921f0b5..679035c5d 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -18,6 +18,10 @@ from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
+def _DUMMY_CALLBACK(response):
+ return response
+
+
class MediaPipeline:
LOG_FAILED_RESULTS = True
@@ -91,7 +95,10 @@ class MediaPipeline:
def _process_request(self, request, info, item):
fp = self._fingerprinter.fingerprint(request)
- cb = request.callback or (lambda _: _)
+ if not request.callback or request.callback is NO_CALLBACK:
+ cb = _DUMMY_CALLBACK
+ else:
+ cb = request.callback
eb = request.errback
request.callback = NO_CALLBACK
request.errback = None