Scrapy, a fast high-level web crawling & scraping framework for Python.
Add source parameter to bytes_received signal
Eugenio Lacuesta6f02a8dccb95
diff --git a/docs/topics/signals.rst b/docs/topics/signals.rstindex f490911f3..3a15bf95c 100644--- a/docs/topics/signals.rst+++ b/docs/topics/signals.rst@@ -162,11 +162,11 @@ bytes_received--------------.. signal:: bytes_received-.. function:: bytes_received(data, request, spider)+.. function:: bytes_received(data, request, spider, source)- Sent by the HTTP 1.1 download handler when a group of bytes is- received for a specific request. This signal might be fired- multiple times for the same request.+ Sent by the HTTP 1.1 and S3 download handlers when a group of bytes is+ received for a specific request. This signal might be fired multiple+ times for the same request, with partial data each time.This signal does not support returning deferreds from its handlers.@@ -179,6 +179,10 @@ bytes_received:param spider: the spider associated with the response:type spider: :class:`~scrapy.spiders.Spider` object+ :param source: a string to identify which handler sent the signal+ (current values could be "http11" or "s3")+ :type source: :class:`str` object+spider_closed-------------diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 6f1bd9ad6..49c9eacac 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -32,8 +32,9 @@ logger = logging.getLogger(__name__)class HTTP11DownloadHandler:lazy = False- def __init__(self, settings, crawler=None):+ def __init__(self, settings, crawler=None, source="http11"):self.crawler = crawler+ self.source = sourceself._pool = HTTPConnectionPool(reactor, persistent=True)self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')self._pool._factory.noisy = False@@ -67,8 +68,8 @@ class HTTP11DownloadHandler:self._disconnect_timeout = 1@classmethod- def from_crawler(cls, crawler):- return cls(crawler.settings, crawler)+ def from_crawler(cls, crawler, **kwargs):+ return cls(crawler.settings, crawler, **kwargs)def download_request(self, request, spider):"""Return a deferred for the HTTP download"""@@ -79,6 +80,7 @@ class HTTP11DownloadHandler:warnsize=getattr(spider, 'download_warnsize', self._default_warnsize),fail_on_dataloss=self._fail_on_dataloss,crawler=self.crawler,+ source=self.source,)return agent.download_request(request)@@ -275,7 +277,7 @@ class ScrapyAgent:_TunnelingAgent = TunnelingAgentdef __init__(self, contextFactory=None, connectTimeout=10, bindAddress=None, pool=None,- maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None):+ maxsize=0, warnsize=0, fail_on_dataloss=True, crawler=None, source=None):self._contextFactory = contextFactoryself._connectTimeout = connectTimeoutself._bindAddress = bindAddress@@ -285,6 +287,7 @@ class ScrapyAgent:self._fail_on_dataloss = fail_on_datalossself._txresponse = Noneself._crawler = crawler+ self._source = sourcedef _get_agent(self, request, timeout):bindaddress = request.meta.get('bindaddress') or self._bindAddress@@ -421,6 +424,7 @@ class ScrapyAgent:warnsize,fail_on_dataloss,self._crawler,+ self._source,))@@ -457,7 +461,9 @@ class _RequestBodyProducer:class _ResponseReader(protocol.Protocol):- def __init__(self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler):+ def __init__(+ self, finished, txresponse, request, maxsize, warnsize, fail_on_dataloss, crawler, source+ ):self._finished = finishedself._txresponse = txresponseself._request = request@@ -469,6 +475,7 @@ class _ResponseReader(protocol.Protocol):self._reached_warnsize = Falseself._bytes_received = 0self._crawler = crawler+ self._source = sourcedef dataReceived(self, bodyBytes):# This maybe called several times after cancel was called with buffered data.@@ -483,6 +490,7 @@ class _ResponseReader(protocol.Protocol):data=bodyBytes,request=self._request,spider=self._crawler.spider,+ source=self._source,)if self._maxsize and self._bytes_received > self._maxsize:diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 40a1fa48e..2366b6394 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -73,6 +73,7 @@ class S3DownloadHandler:objcls=httpdownloadhandler,settings=settings,crawler=crawler,+ source="s3",)self._download_http = _http_handler.download_requestdiff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 8d95d7cac..22a813647 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -730,6 +730,9 @@ class Http11ProxyTestCase(HttpProxyTestCase):class HttpDownloadHandlerMock:+ def __init__(self, *args, **kwargs):+ pass+def download_request(self, request, spider):return requestdiff --git a/tests/test_engine.py b/tests/test_engine.pyindex 3c5cc403b..c83a23b55 100644--- a/tests/test_engine.py+++ b/tests/test_engine.py@@ -112,6 +112,7 @@ class CrawlerRun(object):self.itemerror = []self.itemresp = []self.bytes = defaultdict(lambda: list())+ self.bytes_source = set()self.signals_caught = {}self.spider_class = spider_class@@ -165,8 +166,9 @@ class CrawlerRun(object):def item_scraped(self, item, spider, response):self.itemresp.append((item, response))- def bytes_received(self, data, request, spider):+ def bytes_received(self, data, request, spider, source):self.bytes[request].append(data)+ self.bytes_source.add(source)def request_scheduled(self, request, spider):self.reqplug.append((request, spider))@@ -279,6 +281,7 @@ class EngineTest(unittest.TestCase):def _assert_bytes_received(self):self.assertEqual(9, len(self.run.bytes))+ self.assertEqual(self.run.bytes_source, set(["http11"]))for request, data in self.run.bytes.items():joined_data = b"".join(data)if self.run.getpath(request.url) == "/":