Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge Twisted’s deprecated HTTPClientFactory into ScrapyHTTPClientFactory (#4165)

Adrián Chaves792d29380e37
diff --cc scrapy/core/downloader/webclient.py
index 355045d74,2a01eaac7..b2b96f1ea
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@@ -88,17 -88,18 +88,18 @@@ class ScrapyHTTPPageGetter(HTTPClient)
self.transport.stopProducing()
self.factory.noPage(
- defer.TimeoutError("Getting %s took longer than %s seconds." %
- (self.factory.url, self.factory.timeout)))
+ defer.TimeoutError("Getting %s took longer than %s seconds."
+ % (self.factory.url, self.factory.timeout)))
- class ScrapyHTTPClientFactory(HTTPClientFactory):
- """Scrapy implementation of the HTTPClientFactory overwriting the
- setUrl method to make use of our Url object that cache the parse
- result.
- """
+ # This class used to inherit from Twisted’s
+ # twisted.web.client.HTTPClientFactory. When that class was deprecated in
+ # Twisted (https://github.com/twisted/twisted/pull/643), we merged its
+ # non-overriden code into this class.
+ class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
+
waiting = 1
noisy = False
followRedirect = False