Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Port code from Twisted’s deprecated HTTPClientFactory into ScrapyHTTPClientFactory

Adrián Chaves0946eb335a28
diff --git a/scrapy/core/downloader/webclient.py b/scrapy/core/downloader/webclient.py
index 3fe13414a..16fd214a3 100644
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@ -1,9 +1,9 @@
from time import time
from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
-from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
@@ -93,18 +93,30 @@ class ScrapyHTTPPageGetter(HTTPClient):
(self.factory.url, self.factory.timeout)))
-class ScrapyHTTPClientFactory(HTTPClientFactory):
- """Scrapy implementation of the HTTPClientFactory overwriting the
- setUrl method to make use of our Url object that cache the parse
- result.
- """
+class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
+
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
+ def _build_response(self, body, request):
+ request.meta['download_latency'] = self.headers_time-self.start_time
+ status = int(self.status)
+ headers = Headers(self.response_headers)
+ respcls = responsetypes.from_args(headers=headers, url=self._url)
+ return respcls(url=self._url, status=status, headers=headers, body=body)
+
+ def _set_connection_attributes(self, request):
+ parsed = urlparse_cached(request)
+ self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+ proxy = request.meta.get('proxy')
+ if proxy:
+ self.scheme, _, self.host, self.port, _ = _parse(proxy)
+ self.path = self.url
+
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface
@@ -139,21 +151,59 @@ class ScrapyHTTPClientFactory(HTTPClientFactory):
elif self.method == b'POST':
self.headers['Content-Length'] = 0
- def _build_response(self, body, request):
- request.meta['download_latency'] = self.headers_time-self.start_time
- status = int(self.status)
- headers = Headers(self.response_headers)
- respcls = responsetypes.from_args(headers=headers, url=self._url)
- return respcls(url=self._url, status=status, headers=headers, body=body)
+ def __repr__(self):
+ return "<%s: %s>" % (self.__class__.__name__, self.url)
- def _set_connection_attributes(self, request):
- parsed = urlparse_cached(request)
- self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
- proxy = request.meta.get('proxy')
- if proxy:
- self.scheme, _, self.host, self.port, _ = _parse(proxy)
- self.path = self.url
+ def _cancelTimeout(self, result, timeoutCall):
+ if timeoutCall.active():
+ timeoutCall.cancel()
+ return result
+
+ def buildProtocol(self, addr):
+ p = ClientFactory.buildProtocol(self, addr)
+ p.followRedirect = self.followRedirect
+ p.afterFoundGet = self.afterFoundGet
+ if self.timeout:
+ timeoutCall = reactor.callLater(self.timeout, p.timeout)
+ self.deferred.addBoth(self._cancelTimeout, timeoutCall)
+ return p
def gotHeaders(self, headers):
self.headers_time = time()
self.response_headers = headers
+
+ def gotStatus(self, version, status, message):
+ """
+ Set the status of the request on us.
+ @param version: The HTTP version.
+ @type version: L{bytes}
+ @param status: The HTTP status code, an integer represented as a
+ bytestring.
+ @type status: L{bytes}
+ @param message: The HTTP status message.
+ @type message: L{bytes}
+ """
+ self.version, self.status, self.message = version, status, message
+
+ def page(self, page):
+ if self.waiting:
+ self.waiting = 0
+ self.deferred.callback(page)
+
+ def noPage(self, reason):
+ if self.waiting:
+ self.waiting = 0
+ self.deferred.errback(reason)
+
+ def clientConnectionFailed(self, _, reason):
+ """
+ When a connection attempt fails, the request cannot be issued. If no
+ result has yet been provided to the result Deferred, provide the
+ connection failure reason as an error result.
+ """
+ if self.waiting:
+ self.waiting = 0
+ # If the connection attempt failed, there is nothing more to
+ # disconnect, so just fire that Deferred now.
+ self._disconnectedDeferred.callback(None)
+ self.deferred.errback(reason)