Scrapy, a fast high-level web crawling & scraping framework for Python.
Download handlers: from_crawler factory method, take crawler instead of settings in __init__
Eugenio Lacuesta3d77f74e4089
diff --git a/scrapy/core/downloader/handlers/__init__.py b/scrapy/core/downloader/handlers/__init__.pyindex 0b55d32fa..e8beb2f5a 100644--- a/scrapy/core/downloader/handlers/__init__.py+++ b/scrapy/core/downloader/handlers/__init__.py@@ -5,7 +5,7 @@ from twisted.internet import deferimport sixfrom scrapy.exceptions import NotSupported, NotConfiguredfrom scrapy.utils.httpobj import urlparse_cached-from scrapy.utils.misc import load_object+from scrapy.utils.misc import create_instance, load_objectfrom scrapy.utils.python import without_none_valuesfrom scrapy import signals@@ -48,7 +48,11 @@ class DownloadHandlers(object):dhcls = load_object(path)if skip_lazy and getattr(dhcls, 'lazy', True):return None- dh = dhcls(self._crawler.settings)+ dh = create_instance(+ dhcls,+ self._crawler.settings,+ self._crawler,+ )except NotConfigured as ex:self._notconfigured[scheme] = str(ex)return Nonediff --git a/scrapy/core/downloader/handlers/datauri.py b/scrapy/core/downloader/handlers/datauri.pyindex 9e5020753..97134e618 100644--- a/scrapy/core/downloader/handlers/datauri.py+++ b/scrapy/core/downloader/handlers/datauri.py@@ -8,9 +8,6 @@ from scrapy.utils.decorators import defersclass DataURIDownloadHandler(object):lazy = False- def __init__(self, settings):- super(DataURIDownloadHandler, self).__init__()-@defersdef download_request(self, request, spider):uri = parse_data_uri(request.url)diff --git a/scrapy/core/downloader/handlers/file.py b/scrapy/core/downloader/handlers/file.pyindex 23f25d28d..d445ba2e1 100644--- a/scrapy/core/downloader/handlers/file.py+++ b/scrapy/core/downloader/handlers/file.py@@ -1,4 +1,5 @@from w3lib.url import file_uri_to_path+from scrapy.responsetypes import responsetypesfrom scrapy.utils.decorators import defers@@ -6,9 +7,6 @@ from scrapy.utils.decorators import defersclass FileDownloadHandler(object):lazy = False- def __init__(self, settings):- pass-@defersdef download_request(self, request, spider):filepath = file_uri_to_path(request.url)diff --git a/scrapy/core/downloader/handlers/ftp.py b/scrapy/core/downloader/handlers/ftp.pyindex 39ed67a1a..7a98361ed 100644--- a/scrapy/core/downloader/handlers/ftp.py+++ b/scrapy/core/downloader/handlers/ftp.py@@ -59,6 +59,7 @@ class ReceivedDataProtocol(Protocol):def close(self):self.body.close() if self.filename else self.body.seek(0)+_CODE_RE = re.compile(r"\d+")@@ -70,10 +71,14 @@ class FTPDownloadHandler(object):"default": 503,}- def __init__(self, settings):- self.default_user = settings['FTP_USER']- self.default_password = settings['FTP_PASSWORD']- self.passive_mode = settings['FTP_PASSIVE_MODE']+ def __init__(self, crawler):+ self.default_user = crawler.settings['FTP_USER']+ self.default_password = crawler.settings['FTP_PASSWORD']+ self.passive_mode = crawler.settings['FTP_PASSIVE_MODE']++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)def download_request(self, request, spider):parsed_url = urlparse_cached(request)diff --git a/scrapy/core/downloader/handlers/http10.py b/scrapy/core/downloader/handlers/http10.pyindex be7298531..ce0801bcc 100644--- a/scrapy/core/downloader/handlers/http10.py+++ b/scrapy/core/downloader/handlers/http10.py@@ -1,6 +1,7 @@"""Download handlers for http and https schemes"""from twisted.internet import reactor+from scrapy.utils.misc import load_object, create_instancefrom scrapy.utils.python import to_unicode@@ -8,10 +9,15 @@ from scrapy.utils.python import to_unicodeclass HTTP10DownloadHandler(object):lazy = False- def __init__(self, settings):- self.HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])- self.ClientContextFactory = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])- self._settings = settings+ def __init__(self, crawler):+ self.HTTPClientFactory = load_object(crawler.settings['DOWNLOADER_HTTPCLIENTFACTORY'])+ self.ClientContextFactory = load_object(crawler.settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])+ self._crawler = crawler+ self._settings = crawler.settings++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)def download_request(self, request, spider):"""Return a deferred for the HTTP download"""@@ -22,7 +28,11 @@ class HTTP10DownloadHandler(object):def _connect(self, factory):host, port = to_unicode(factory.host), factory.portif factory.scheme == b'https':- client_context_factory = create_instance(self.ClientContextFactory, settings=self._settings, crawler=None)+ client_context_factory = create_instance(+ self.ClientContextFactory,+ settings=self._settings,+ crawler=self._crawler,+ )return reactor.connectSSL(host, port, factory, client_context_factory)else:return reactor.connectTCP(host, port, factory)diff --git a/scrapy/core/downloader/handlers/http11.py b/scrapy/core/downloader/handlers/http11.pyindex 7d917cb74..b424a7999 100644--- a/scrapy/core/downloader/handlers/http11.py+++ b/scrapy/core/downloader/handlers/http11.py@@ -30,7 +30,9 @@ logger = logging.getLogger(__name__)class HTTP11DownloadHandler(object):lazy = False- def __init__(self, settings):+ def __init__(self, crawler):+ settings = crawler.settings+self._pool = HTTPConnectionPool(reactor, persistent=True)self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')self._pool._factory.noisy = False@@ -42,7 +44,7 @@ class HTTP11DownloadHandler(object):self._contextFactory = create_instance(self._contextFactoryClass,settings=settings,- crawler=None,+ crawler=crawler,method=self._sslMethod,)except TypeError:@@ -50,7 +52,7 @@ class HTTP11DownloadHandler(object):self._contextFactory = create_instance(self._contextFactoryClass,settings=settings,- crawler=None,+ crawler=crawler,)msg = """'%s' does not accept `method` argument (type OpenSSL.SSL method,\@@ -63,6 +65,10 @@ class HTTP11DownloadHandler(object):self._fail_on_dataloss = settings.getbool('DOWNLOAD_FAIL_ON_DATALOSS')self._disconnect_timeout = 1+ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)+def download_request(self, request, spider):"""Return a deferred for the HTTP download"""agent = ScrapyAgent(diff --git a/scrapy/core/downloader/handlers/s3.py b/scrapy/core/downloader/handlers/s3.pyindex 808d1bf21..220296fb3 100644--- a/scrapy/core/downloader/handlers/s3.py+++ b/scrapy/core/downloader/handlers/s3.py@@ -32,13 +32,12 @@ def _get_boto_connection():class S3DownloadHandler(object):- def __init__(self, settings, aws_access_key_id=None, aws_secret_access_key=None, \- httpdownloadhandler=HTTPDownloadHandler, **kw):-+ def __init__(self, crawler, aws_access_key_id=None, aws_secret_access_key=None,+ httpdownloadhandler=HTTPDownloadHandler, **kw):if not aws_access_key_id:- aws_access_key_id = settings['AWS_ACCESS_KEY_ID']+ aws_access_key_id = crawler.settings['AWS_ACCESS_KEY_ID']if not aws_secret_access_key:- aws_secret_access_key = settings['AWS_SECRET_ACCESS_KEY']+ aws_secret_access_key = crawler.settings['AWS_SECRET_ACCESS_KEY']# If no credentials could be found anywhere,# consider this an anonymous connection request by default;@@ -67,7 +66,11 @@ class S3DownloadHandler(object):except Exception as ex:raise NotConfigured(str(ex))- self._download_http = httpdownloadhandler(settings).download_request+ self._download_http = httpdownloadhandler(crawler).download_request++ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)def download_request(self, request, spider):p = urlparse_cached(request)diff --git a/tests/test_downloader_handlers.py b/tests/test_downloader_handlers.pyindex 60124b93f..815342219 100644--- a/tests/test_downloader_handlers.py+++ b/tests/test_downloader_handlers.py@@ -30,7 +30,6 @@ from scrapy.spiders import Spiderfrom scrapy.http import Headers, Requestfrom scrapy.http.response.text import TextResponsefrom scrapy.responsetypes import responsetypes-from scrapy.settings import Settingsfrom scrapy.utils.test import get_crawler, skip_if_no_botofrom scrapy.utils.python import to_bytesfrom scrapy.exceptions import NotConfigured@@ -45,6 +44,10 @@ class DummyDH(object):def __init__(self, crawler):pass+ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)+class DummyLazyDH(object):# Default is lazy for backward compatibility@@ -52,6 +55,10 @@ class DummyLazyDH(object):def __init__(self, crawler):pass+ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)+class OffDH(object):lazy = False@@ -59,6 +66,10 @@ class OffDH(object):def __init__(self, crawler):raise NotConfigured+ @classmethod+ def from_crawler(cls, crawler):+ return cls(crawler)+class LoadTestCase(unittest.TestCase):@@ -106,7 +117,7 @@ class FileTestCase(unittest.TestCase):self.tmpname = self.mktemp()with open(self.tmpname + '^', 'w') as f:f.write('0123456789')- self.download_request = FileDownloadHandler(Settings()).download_request+ self.download_request = FileDownloadHandler().download_requestdef tearDown(self):os.unlink(self.tmpname + '^')@@ -239,7 +250,7 @@ class HttpTestCase(unittest.TestCase):else:self.port = reactor.listenTCP(0, self.wrapper, interface=self.host)self.portno = self.port.getHost().port- self.download_handler = self.download_handler_cls(Settings())+ self.download_handler = self.download_handler_cls(get_crawler())self.download_request = self.download_handler.download_request@defer.inlineCallbacks@@ -479,9 +490,9 @@ class Http11TestCase(HttpTestCase):return self.test_download_broken_content_allow_data_loss('broken-chunked')def test_download_broken_content_allow_data_loss_via_setting(self, url='broken'):- download_handler = self.download_handler_cls(Settings({- 'DOWNLOAD_FAIL_ON_DATALOSS': False,- }))+ download_handler = self.download_handler_cls(+ get_crawler(settings_dict={'DOWNLOAD_FAIL_ON_DATALOSS': False})+ )request = Request(self.getURL(url))d = download_handler.download_request(request, Spider('foo'))d.addCallback(lambda r: r.flags)@@ -499,9 +510,9 @@ class Https11TestCase(Http11TestCase):@defer.inlineCallbacksdef test_tls_logging(self):- download_handler = self.download_handler_cls(Settings({- 'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True,- }))+ download_handler = self.download_handler_cls(+ get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_VERBOSE_LOGGING': True})+ )try:with LogCapture() as log_capture:request = Request(self.getURL('file'))@@ -569,7 +580,8 @@ class Https11CustomCiphers(unittest.TestCase):interface=self.host)self.portno = self.port.getHost().portself.download_handler = self.download_handler_cls(- Settings({'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'}))+ get_crawler(settings_dict={'DOWNLOADER_CLIENT_TLS_CIPHERS': 'CAMELLIA256-SHA'})+ )self.download_request = self.download_handler.download_request@defer.inlineCallbacks@@ -665,7 +677,7 @@ class HttpProxyTestCase(unittest.TestCase):wrapper = WrappingFactory(site)self.port = reactor.listenTCP(0, wrapper, interface='127.0.0.1')self.portno = self.port.getHost().port- self.download_handler = self.download_handler_cls(Settings())+ self.download_handler = self.download_handler_cls(get_crawler())self.download_request = self.download_handler.download_request@defer.inlineCallbacks@@ -738,9 +750,10 @@ class S3AnonTestCase(unittest.TestCase):def setUp(self):skip_if_no_boto()- self.s3reqh = S3DownloadHandler(Settings(),- httpdownloadhandler=HttpDownloadHandlerMock,- #anon=True, # is implicit+ self.s3reqh = S3DownloadHandler(+ crawler=get_crawler(),+ httpdownloadhandler=HttpDownloadHandlerMock,+ #anon=True, # is implicit)self.download_request = self.s3reqh.download_requestself.spider = Spider('foo')@@ -766,9 +779,12 @@ class S3TestCase(unittest.TestCase):def setUp(self):skip_if_no_boto()- s3reqh = S3DownloadHandler(Settings(), self.AWS_ACCESS_KEY_ID,- self.AWS_SECRET_ACCESS_KEY,- httpdownloadhandler=HttpDownloadHandlerMock)+ s3reqh = S3DownloadHandler(+ get_crawler(),+ self.AWS_ACCESS_KEY_ID,+ self.AWS_SECRET_ACCESS_KEY,+ httpdownloadhandler=HttpDownloadHandlerMock,+ )self.download_request = s3reqh.download_requestself.spider = Spider('foo')@@ -788,7 +804,7 @@ class S3TestCase(unittest.TestCase):def test_extra_kw(self):try:- S3DownloadHandler(Settings(), extra_kw=True)+ S3DownloadHandler(get_crawler(), extra_kw=True)except Exception as e:self.assertIsInstance(e, (TypeError, NotConfigured))else:@@ -928,7 +944,7 @@ class BaseFTPTestCase(unittest.TestCase):self.factory = FTPFactory(portal=p)self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")self.portNum = self.port.getHost().port- self.download_handler = FTPDownloadHandler(Settings())+ self.download_handler = FTPDownloadHandler(get_crawler())self.addCleanup(self.port.stopListening)def tearDown(self):@@ -1042,7 +1058,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):userAnonymous=self.username)self.port = reactor.listenTCP(0, self.factory, interface="127.0.0.1")self.portNum = self.port.getHost().port- self.download_handler = FTPDownloadHandler(Settings())+ self.download_handler = FTPDownloadHandler(get_crawler())self.addCleanup(self.port.stopListening)def tearDown(self):@@ -1052,7 +1068,7 @@ class AnonymousFTPTestCase(BaseFTPTestCase):class DataURITestCase(unittest.TestCase):def setUp(self):- self.download_handler = DataURIDownloadHandler(Settings())+ self.download_handler = DataURIDownloadHandler()self.download_request = self.download_handler.download_requestself.spider = Spider('foo')