Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Merge branch 'master' into remove-deprecated-http-client-factory

Adrián Chavesf2bbdb43ee76
diff --cc scrapy/core/downloader/webclient.py
index 26726afd4,a90a77b2b..2a01eaac7
--- a/scrapy/core/downloader/webclient.py
+++ b/scrapy/core/downloader/webclient.py
@@@ -1,9 -1,9 +1,9 @@@
from time import time
- from six.moves.urllib.parse import urlparse, urlunparse, urldefrag
+ from urllib.parse import urlparse, urlunparse, urldefrag
-from twisted.web.client import HTTPClientFactory
from twisted.web.http import HTTPClient
-from twisted.internet import defer
+from twisted.internet import defer, reactor
+from twisted.internet.protocol import ClientFactory
from scrapy.http import Headers
from scrapy.utils.httpobj import urlparse_cached
@@@ -88,39 -87,23 +87,39 @@@ class ScrapyHTTPPageGetter(HTTPClient)
if self.factory.url.startswith(b'https'):
self.transport.stopProducing()
- self.factory.noPage(\
- defer.TimeoutError("Getting %s took longer than %s seconds." % \
- (self.factory.url, self.factory.timeout)))
+ self.factory.noPage(
+ defer.TimeoutError("Getting %s took longer than %s seconds." %
+ (self.factory.url, self.factory.timeout)))
-class ScrapyHTTPClientFactory(HTTPClientFactory):
- """Scrapy implementation of the HTTPClientFactory overwriting the
- setUrl method to make use of our Url object that cache the parse
- result.
- """
+# This class used to inherit from Twisted’s
+# twisted.web.client.HTTPClientFactory. When that class was deprecated in
+# Twisted (https://github.com/twisted/twisted/pull/643), we merged its
+# non-overriden code into this class.
+class ScrapyHTTPClientFactory(ClientFactory):
protocol = ScrapyHTTPPageGetter
+
waiting = 1
noisy = False
followRedirect = False
afterFoundGet = False
+ def _build_response(self, body, request):
- request.meta['download_latency'] = self.headers_time-self.start_time
++ request.meta['download_latency'] = self.headers_time - self.start_time
+ status = int(self.status)
+ headers = Headers(self.response_headers)
+ respcls = responsetypes.from_args(headers=headers, url=self._url)
+ return respcls(url=self._url, status=status, headers=headers, body=body)
+
+ def _set_connection_attributes(self, request):
+ parsed = urlparse_cached(request)
+ self.scheme, self.netloc, self.host, self.port, self.path = _parsed_url_args(parsed)
+ proxy = request.meta.get('proxy')
+ if proxy:
+ self.scheme, _, self.host, self.port, _ = _parse(proxy)
+ self.path = self.url
+
def __init__(self, request, timeout=180):
self._url = urldefrag(request.url)[0]
# converting to bytes to comply to Twisted interface