Scrapy, a fast high-level web crawling & scraping framework for Python.
Explicitly call Twisted transport stopProducing() on HTTP/1.1 timeouts
Paul Tremberthecddc093a4b9
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 5261f542b..6fc2bd2bd 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -193,6 +193,7 @@ class ScrapyAgent(object):self._pool = poolself._maxsize = maxsizeself._warnsize = warnsize+ self._txresponse = Nonedef _get_agent(self, request, timeout):bindaddress = request.meta.get('bindaddress') or self._bindAddress@@ -259,6 +260,11 @@ class ScrapyAgent(object):if self._timeout_cl.active():self._timeout_cl.cancel()return result+ # needed for HTTPS requests, otherwise _ResponseReader doesn't+ # receive connectionLost()+ if self._txresponse:+ self._txresponse._transport.stopProducing()+raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))def _cb_latency(self, result, request, start_time):@@ -294,6 +300,10 @@ class ScrapyAgent(object):d = defer.Deferred(_cancel)txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))++ # save response for timeouts+ self._txresponse = txresponse+return ddef _cb_bodydone(self, result, request, url):diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 88b60e4ff..e13becf1b 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -182,17 +182,19 @@ class HttpTestCase(unittest.TestCase):return d@defer.inlineCallbacks- def test_timeout_download_from_spider(self):- if self.scheme == 'https':- raise unittest.SkipTest(- 'test_timeout_download_from_spider skipped under https')+ def test_timeout_download_from_spider_nodata_rcvd(self):+ # client connects but no data is receivedspider = Spider('foo')meta = {'download_timeout': 0.2}- # client connects but no data is receivedrequest = Request(self.getURL('wait'), meta=meta)d = self.download_request(request, spider)yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)++ @defer.inlineCallbacks+ def test_timeout_download_from_spider_server_hangs(self):# client connects, server send headers and some body bytes but hangs+ spider = Spider('foo')+ meta = {'download_timeout': 0.2}request = Request(self.getURL('hang-after-headers'), meta=meta)d = self.download_request(request, spider)yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)