Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Don't run callbacks of requests from get_media_requests(). (#6386)

Andrey Rakhmatullin492c3bce9dfc
diff --git a/scrapy/pipelines/media.py b/scrapy/pipelines/media.py
index 25e00b0ea..0e374265e 100644
--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@@ -24,10 +24,6 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
-def _DUMMY_CALLBACK(response):
- return response
-
-
class MediaPipeline(ABC):
LOG_FAILED_RESULTS = True
@@ -89,10 +85,6 @@ class MediaPipeline(ABC):
def _process_request(self, request, info, item):
fp = self._fingerprinter.fingerprint(request)
- if not request.callback or request.callback is NO_CALLBACK:
- cb = _DUMMY_CALLBACK
- else:
- cb = request.callback
eb = request.errback
request.callback = NO_CALLBACK
request.errback = None
@@ -100,14 +92,12 @@ class MediaPipeline(ABC):
# Return cached result if request was already seen
if fp in info.downloaded:
d = defer_result(info.downloaded[fp])
- d.addCallback(cb)
if eb:
d.addErrback(eb)
return d
# Otherwise, wait for result
wad = Deferred()
- wad.addCallback(cb)
if eb:
wad.addErrback(eb)
info.waiting[fp].append(wad)
diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py
index 763453551..127775f43 100644
--- a/tests/test_pipeline_media.py
+++ b/tests/test_pipeline_media.py
@@ -211,10 +211,6 @@ class MockedMediaPipeline(UserDefinedPipeline):
class MediaPipelineTestCase(BaseMediaPipelineTestCase):
pipeline_class = MockedMediaPipeline
- def _callback(self, result):
- self.pipe._mockcalled.append("request_callback")
- return result
-
def _errback(self, result):
self.pipe._mockcalled.append("request_errback")
return result
@@ -225,7 +221,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
req = Request(
"http://url1",
meta={"response": rsp},
- callback=self._callback,
errback=self._errback,
)
item = {"requests": req}
@@ -237,7 +232,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
"get_media_requests",
"media_to_download",
"media_downloaded",
- "request_callback",
"item_completed",
],
)
@@ -249,7 +243,6 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
req = Request(
"http://url1",
meta={"response": fail},
- callback=self._callback,
errback=self._errback,
)
item = {"requests": req}