Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Explicitly call Twisted transport stopProducing() on HTTP/1.0 timeouts

Paul Tremberthc9e781357da2
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 9bcc51943..1c89a0f9e 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -83,6 +83,11 @@ class ScrapyHTTPPageGetter(HTTPClient):
def timeout(self):
self.transport.loseConnection()
+
+ # transport cleanup needed for HTTPS connections
+ if self.factory.url.startswith(b'https'):
+ self.transport.stopProducing()
+
self.factory.noPage(\
defer.TimeoutError("Getting %s took longer than %s seconds." % \
(self.factory.url, self.factory.timeout)))