Scrapy, a fast high-level web crawling & scraping framework for Python.
Port code from Twisted’s deprecated HTTPClientFactory into ScrapyHTTPClientFactory
Adrián Chaves0946eb335a28
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.pyindex 3fe13414a..16fd214a3 100644--- a/scrapy/core/downloader/webclient.py+++ b/scrapy/core/downloader/webclient.py@@ -1,9 +1,9 @@from time import timefrom six.moves.urllib.parse import urlparse, urlunparse, urldefrag-from twisted.web.client import HTTPClientFactoryfrom twisted.web.http import HTTPClient-from twisted.internet import defer+from twisted.internet import defer, reactor+from twisted.internet.protocol import ClientFactoryfrom scrapy.http import Headersfrom scrapy.utils.httpobj import urlparse_cached@@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient):(self.factory.url, self.factory.timeout)))-class ScrapyHTTPClientFactory(HTTPClientFactory):- """Scrapy implementation of the HTTPClientFactory overwriting the- setUrl method to make use of our Url object that cache the parse- result.- """+class ScrapyHTTPClientFactory(ClientFactory):protocol = ScrapyHTTPPageGetter+waiting = 1noisy = FalsefollowRedirect = FalseafterFoundGet = False+ def _build_response(self, body, request):+ request.meta['download_latency'] = self.headers_time-self.start_time+ status = int(self.status)+ headers = Headers(self.response_headers)+ respcls = responsetypes.from_args(headers=headers, url=self._url)+ return respcls(url=self._url, status=status, headers=headers, body=body)++ def _set_connection_attributes(self, request):+ parsed = urlparse_cached(request)+ self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)+ proxy = request.meta.get('proxy')+ if proxy:+ self.scheme, _, self.host, self.port, _ = _parse(proxy)+ self.path = self.url+def __init__(self, request, timeout=180):self._url = urldefrag(request.url)[0]# converting to bytes to comply to Twisted interface@@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):elif self.method == b'POST':self.headers['Content-Length'] = 0- def _build_response(self, body, request):- request.meta['download_latency'] = self.headers_time-self.start_time- status = int(self.status)- headers = Headers(self.response_headers)- respcls = responsetypes.from_args(headers=headers, url=self._url)- return respcls(url=self._url, status=status, headers=headers, body=body)+ def __repr__(self):+ return "<%s: %s>" % (self.__class__.__name__, self.url)- def _set_connection_attributes(self, request):- parsed = urlparse_cached(request)- self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)- proxy = request.meta.get('proxy')- if proxy:- self.scheme, _, self.host, self.port, _ = _parse(proxy)- self.path = self.url+ def _cancelTimeout(self, result, timeoutCall):+ if timeoutCall.active():+ timeoutCall.cancel()+ return result++ def buildProtocol(self, addr):+ p = ClientFactory.buildProtocol(self, addr)+ p.followRedirect = self.followRedirect+ p.afterFoundGet = self.afterFoundGet+ if self.timeout:+ timeoutCall = reactor.callLater(self.timeout, p.timeout)+ self.deferred.addBoth(self._cancelTimeout, timeoutCall)+ return pdef gotHeaders(self, headers):self.headers_time = time()self.response_headers = headers++ def gotStatus(self, version, status, message):+ """+ Set the status of the request on us.+ @param version: The HTTP version.+ @type version: L{bytes}+ @param status: The HTTP status code, an integer represented as a+ bytestring.+ @type status: L{bytes}+ @param message: The HTTP status message.+ @type message: L{bytes}+ """+ self.version, self.status, self.message = version, status, message++ def page(self, page):+ if self.waiting:+ self.waiting = 0+ self.deferred.callback(page)++ def noPage(self, reason):+ if self.waiting:+ self.waiting = 0+ self.deferred.errback(reason)++ def clientConnectionFailed(self, _, reason):+ """+ When a connection attempt fails, the request cannot be issued. If no+ result has yet been provided to the result Deferred, provide the+ connection failure reason as an error result.+ """+ if self.waiting:+ self.waiting = 0+ # If the connection attempt failed, there is nothing more to+ # disconnect, so just fire that Deferred now.+ self._disconnectedDeferred.callback(None)+ self.deferred.errback(reason)