Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Pass callback results as dicts instead of tuples

Eugenio Lacuesta91a78eef3ee9
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 190ae1d3b..e904cbc05 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -384,7 +384,13 @@ class ScrapyAgent(object):
def _cb_bodyready(self, txresponse, request):
# deliverBody hangs for responses without body
if txresponse.length == 0:
- return txresponse, b'', None, None
+ return {
+ "txresponse": txresponse,
+ "body": b"",
+ "flags": None,
+ "certificate": None,
+ "ip_address": None,
+ }
maxsize = request.meta.get('download_maxsize', self._maxsize)
warnsize = request.meta.get('download_warnsize', self._warnsize)
@@ -420,12 +426,18 @@ class ScrapyAgent(object):
return d
def _cb_bodydone(self, result, request, url):
- txresponse, body, flags, certificate, ip_address = result
- status = int(txresponse.code)
- headers = Headers(txresponse.headers.getAllRawHeaders())
- respcls = responsetypes.from_args(headers=headers, url=url, body=body)
- return respcls(url=url, status=status, headers=headers, body=body,
- flags=flags, certificate=certificate, ip_address=ip_address)
+ status = int(result["txresponse"].code)
+ headers = Headers(result["txresponse"].headers.getAllRawHeaders())
+ respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])
+ return respcls(
+ url=url,
+ status=status,
+ headers=headers,
+ body=result["body"],
+ flags=result["flags"],
+ certificate=result["certificate"],
+ ip_address=result["ip_address"],
+ )
@implementer(IBodyProducer)
@@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol):
body = self._bodybuf.getvalue()
if reason.check(ResponseDone):
- self._finished.callback(
- (self._txresponse, body, None, self._certificate, self._ip_address)
- )
+ self._finished.callback({
+ "txresponse": self._txresponse,
+ "body": body,
+ "flags": None,
+ "certificate": self._certificate,
+ "ip_address": self._ip_address,
+ })
return
if reason.check(PotentialDataLoss):
- self._finished.callback(
- (self._txresponse, body, ['partial'], self._certificate, self._ip_address)
- )
+ self._finished.callback({
+ "txresponse": self._txresponse,
+ "body": body,
+ "flags": ["partial"],
+ "certificate": self._certificate,
+ "ip_address": self._ip_address,
+ })
return
if reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):
if not self._fail_on_dataloss:
- self._finished.callback(
- (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)
- )
+ self._finished.callback({
+ "txresponse": self._txresponse,
+ "body": body,
+ "flags": ["dataloss"],
+ "certificate": self._certificate,
+ "ip_address": self._ip_address,
+ })
return
elif not self._fail_on_dataloss_warned: