Scrapy, a fast high-level web crawling & scraping framework for Python.
Explicitly call Twisted transport stopProducing() on HTTP/1.0 timeouts
Paul Tremberthc9e781357da2
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 9bcc51943..1c89a0f9e 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -83,6 +83,11 @@ class ScrapyHTTPPageGetter(HTTPClient):def timeout(self):self.transport.loseConnection()++ # transport cleanup needed for HTTPS connections+ if self.factory.url.startswith(b'https'):+ self.transport.stopProducing()+self.factory.noPage(\defer.TimeoutError("Getting %s took longer than %s seconds." % \(self.factory.url, self.factory.timeout)))