Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Explicitly call Twisted transport stopProducing() on HTTP/1.1 timeouts

Paul Tremberthecddc093a4b9
diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.py
index 5261f542b..6fc2bd2bd 100644
--- a/scrapy/core/downloader/handlers/http11.py
+++ b/scrapy/core/downloader/handlers/http11.py
@@ -193,6 +193,7 @@ class ScrapyAgent(object):
self._pool = pool
self._maxsize = maxsize
self._warnsize = warnsize
+ self._txresponse = None
def _get_agent(self, request, timeout):
bindaddress = request.meta.get('bindaddress') or self._bindAddress
@@ -259,6 +260,11 @@ class ScrapyAgent(object):
if self._timeout_cl.active():
self._timeout_cl.cancel()
return result
+ # needed for HTTPS requests, otherwise _ResponseReader doesn't
+ # receive connectionLost()
+ if self._txresponse:
+ self._txresponse._transport.stopProducing()
+
raise TimeoutError("Getting %s took longer than %s seconds." % (url, timeout))
def _cb_latency(self, result, request, start_time):
@@ -294,6 +300,10 @@ class ScrapyAgent(object):
d = defer.Deferred(_cancel)
txresponse.deliverBody(_ResponseReader(d, txresponse, request, maxsize, warnsize))
+
+ # save response for timeouts
+ self._txresponse = txresponse
+
return d
def _cb_bodydone(self, result, request, url):
diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.py
index 88b60e4ff..e13becf1b 100644
--- a/tests/test_downloader_handlers.py
+++ b/tests/test_downloader_handlers.py
@@ -182,17 +182,19 @@ class HttpTestCase(unittest.TestCase):
return d
@defer.inlineCallbacks
- def test_timeout_download_from_spider(self):
- if self.scheme == 'https':
- raise unittest.SkipTest(
- 'test_timeout_download_from_spider skipped under https')
+ def test_timeout_download_from_spider_nodata_rcvd(self):
+ # client connects but no data is received
spider = Spider('foo')
meta = {'download_timeout': 0.2}
- # client connects but no data is received
request = Request(self.getURL('wait'), meta=meta)
d = self.download_request(request, spider)
yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)
+
+ @defer.inlineCallbacks
+ def test_timeout_download_from_spider_server_hangs(self):
# client connects, server send headers and some body bytes but hangs
+ spider = Spider('foo')
+ meta = {'download_timeout': 0.2}
request = Request(self.getURL('hang-after-headers'), meta=meta)
d = self.download_request(request, spider)
yield self.assertFailure(d, defer.TimeoutError, error.TimeoutError)