Scrapy, a fast high-level web crawling & scraping framework for Python.
Pass callback results as dicts instead of tuples
Eugenio Lacuesta91a78eef3ee9
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 190ae1d3b..e904cbc05 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -384,7 +384,13 @@ class ScrapyAgent(object):def _cb_bodyready(self, txresponse, request):# deliverBody hangs for responses without bodyif txresponse.length == 0:- return txresponse, b'', None, None+ return {+ "txresponse": txresponse,+ "body": b"",+ "flags": None,+ "certificate": None,+ "ip_address": None,+ }maxsize = request.meta.get('download_maxsize', self._maxsize)warnsize = request.meta.get('download_warnsize', self._warnsize)@@ -420,12 +426,18 @@ class ScrapyAgent(object):return ddef _cb_bodydone(self, result, request, url):- txresponse, body, flags, certificate, ip_address = result- status = int(txresponse.code)- headers = Headers(txresponse.headers.getAllRawHeaders())- respcls = responsetypes.from_args(headers=headers, url=url, body=body)- return respcls(url=url, status=status, headers=headers, body=body,- flags=flags, certificate=certificate, ip_address=ip_address)+ status = int(result["txresponse"].code)+ headers = Headers(result["txresponse"].headers.getAllRawHeaders())+ respcls = responsetypes.from_args(headers=headers, url=url, body=result["body"])+ return respcls(+ url=url,+ status=status,+ headers=headers,+ body=result["body"],+ flags=result["flags"],+ certificate=result["certificate"],+ ip_address=result["ip_address"],+ )@implementer(IBodyProducer)@@ -501,22 +513,34 @@ class _ResponseReader(protocol.Protocol):body = self._bodybuf.getvalue()if reason.check(ResponseDone):- self._finished.callback(- (self._txresponse, body, None, self._certificate, self._ip_address)- )+ self._finished.callback({+ "txresponse": self._txresponse,+ "body": body,+ "flags": None,+ "certificate": self._certificate,+ "ip_address": self._ip_address,+ })returnif reason.check(PotentialDataLoss):- self._finished.callback(- (self._txresponse, body, ['partial'], self._certificate, self._ip_address)- )+ self._finished.callback({+ "txresponse": self._txresponse,+ "body": body,+ "flags": ["partial"],+ "certificate": self._certificate,+ "ip_address": self._ip_address,+ })returnif reason.check(ResponseFailed) and any(r.check(_DataLoss) for r in reason.value.reasons):if not self._fail_on_dataloss:- self._finished.callback(- (self._txresponse, body, ['dataloss'], self._certificate, self._ip_address)- )+ self._finished.callback({+ "txresponse": self._txresponse,+ "body": body,+ "flags": ["dataloss"],+ "certificate": self._certificate,+ "ip_address": self._ip_address,+ })returnelif not self._fail_on_dataloss_warned: