Scrapy, a fast high-level web crawling & scraping framework for Python.
Merge branch 'master' into remove-deprecated-http-client-factory
Adrián Chavesf2bbdb43ee76
diff --cc scrapy/core/downloader/webclient.pyindex 26726afd4,a90a77b2b..2a01eaac7--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@@ -1,9 -1,9 +1,9 @@@from time import time- from six.moves.urllib.parse import urlparse, urlunparse, urldefrag+ from urllib.parse import urlparse, urlunparse, urldefrag-from twisted.web.client import HTTPClientFactoryfrom twisted.web.http import HTTPClient-from twisted.internet import defer+from twisted.internet import defer, reactor+from twisted.internet.protocol import ClientFactoryfrom scrapy.http import Headersfrom scrapy.utils.httpobj import urlparse_cached@@@ -88,39 -87,23 +87,39 @@@ class ScrapyHTTPPageGetter(HTTPClient)if self.factory.url.startswith(b'https'):self.transport.stopProducing()- self.factory.noPage(\- defer.TimeoutError("Getting %s took longer than %s seconds." % \- (self.factory.url, self.factory.timeout)))+ self.factory.noPage(+ defer.TimeoutError("Getting %s took longer than %s seconds." %+ (self.factory.url, self.factory.timeout)))-class ScrapyHTTPClientFactory(HTTPClientFactory):- """Scrapy implementation of the HTTPClientFactory overwriting the- setUrl method to make use of our Url object that cache the parse- result.- """+# This class used to inherit from Twisted’s+# twisted.web.client.HTTPClientFactory. When that class was deprecated in+# Twisted (https://github.com/twisted/twisted/pull/643), we merged its+# non-overriden code into this class.+class ScrapyHTTPClientFactory(ClientFactory):protocol = ScrapyHTTPPageGetter+waiting = 1noisy = FalsefollowRedirect = FalseafterFoundGet = False+ def _build_response(self, body, request):- request.meta['download_latency'] = self.headers_time-self.start_time++ request.meta['download_latency'] = self.headers_time - self.start_time+ status = int(self.status)+ headers = Headers(self.response_headers)+ respcls = responsetypes.from_args(headers=headers, url=self._url)+ return respcls(url=self._url, status=status, headers=headers, body=body)++ def _set_connection_attributes(self, request):+ parsed = urlparse_cached(request)+ self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)+ proxy = request.meta.get('proxy')+ if proxy:+ self.scheme, _, self.host, self.port, _ = _parse(proxy)+ self.path = self.url+def __init__(self, request, timeout=180):self._url = urldefrag(request.url)[0]# converting to bytes to comply to Twisted interface