Scrapy, a fast high-level web crawling & scraping framework for Python.
Add HTTPPROXY_ENABLED setting (default True)
Eugenio Lacuestaae0ea31abd4e
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex f0ff3c77c..0ef3fb071 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -681,10 +681,9 @@ HttpProxyMiddleware* ``no_proxy``You can also set the meta key ``proxy`` per-request, to a value like- ``http://username:password@some_proxy_server:port``. Keep in mind- this value will take precedence over ``http_proxy``/``https_proxy``- environment variables, and it will also ignore ``no_proxy`` environment- variable.+ ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.+ Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``+ environment variables, and it will also ignore ``no_proxy`` environment variable... _urllib: https://docs.python.org/2/library/urllib.html.. _urllib2: https://docs.python.org/2/library/urllib2.html@@ -952,8 +951,16 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.HttpProxyMiddleware settings~~~~~~~~~~~~~~~~~~~~~~~~~~~~+.. setting:: HTTPPROXY_ENABLED.. setting:: HTTPPROXY_AUTH_ENCODING+HTTPPROXY_ENABLED+^^^^^^^^^^^^^^^^^++Default: ``True``++Whether or not to enable the :class:`HttpProxyMiddleware`.+HTTPPROXY_AUTH_ENCODING^^^^^^^^^^^^^^^^^^^^^^^diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex edc1c52ed..0d5320bf8 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -8,6 +8,7 @@ except ImportError:from six.moves.urllib.parse import urlunparsefrom scrapy.utils.httpobj import urlparse_cached+from scrapy.exceptions import NotConfiguredfrom scrapy.utils.python import to_bytes@@ -21,6 +22,8 @@ class HttpProxyMiddleware(object):@classmethoddef from_crawler(cls, crawler):+ if not crawler.settings.getbool('HTTPPROXY_ENABLED'):+ raise NotConfiguredauth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')return cls(auth_encoding)diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex 24714a7a8..cb88bc2bf 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'HTTPCACHE_GZIP = False+HTTPPROXY_ENABLED = TrueHTTPPROXY_AUTH_ENCODING = 'latin-1'IMAGES_STORE_S3_ACL = 'private'diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex dd09e4dd0..c77179ceb 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -1,11 +1,14 @@import osimport sys+from functools import partialfrom twisted.trial.unittest import TestCase, SkipTestfrom scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddlewarefrom scrapy.exceptions import NotConfiguredfrom scrapy.http import Response, Requestfrom scrapy.spiders import Spider+from scrapy.crawler import Crawler+from scrapy.settings import Settingsspider = Spider('foo')@@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase):def tearDown(self):os.environ = self._oldenv+ def test_not_enabled(self):+ settings = Settings({'HTTPPROXY_ENABLED': False})+ crawler = Crawler(spider, settings)+ self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))+def test_no_enviroment_proxies(self):os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}mw = HttpProxyMiddleware()