Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Add HTTPPROXY_ENABLED setting (default True)

Eugenio Lacuestaae0ea31abd4e
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index f0ff3c77c..0ef3fb071 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -681,10 +681,9 @@ HttpProxyMiddleware
* ``no_proxy``
You can also set the meta key ``proxy`` per-request, to a value like
- ``http://username:password@some_proxy_server:port``. Keep in mind
- this value will take precedence over ``http_proxy``/``https_proxy``
- environment variables, and it will also ignore ``no_proxy`` environment
- variable.
+ ``http://some_proxy_server:port`` or ``http://username:password@some_proxy_server:port``.
+ Keep in mind this value will take precedence over ``http_proxy``/``https_proxy``
+ environment variables, and it will also ignore ``no_proxy`` environment variable.
.. _urllib: https://docs.python.org/2/library/urllib.html
.. _urllib2: https://docs.python.org/2/library/urllib2.html
@@ -952,8 +951,16 @@ enable it for :ref:`broad crawls <topics-broad-crawls>`.
HttpProxyMiddleware settings
~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+.. setting:: HTTPPROXY_ENABLED
.. setting:: HTTPPROXY_AUTH_ENCODING
+HTTPPROXY_ENABLED
+^^^^^^^^^^^^^^^^^
+
+Default: ``True``
+
+Whether or not to enable the :class:`HttpProxyMiddleware`.
+
HTTPPROXY_AUTH_ENCODING
^^^^^^^^^^^^^^^^^^^^^^^
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index edc1c52ed..0d5320bf8 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -8,6 +8,7 @@ except ImportError:
from six.moves.urllib.parse import urlunparse
from scrapy.utils.httpobj import urlparse_cached
+from scrapy.exceptions import NotConfigured
from scrapy.utils.python import to_bytes
@@ -21,6 +22,8 @@ class HttpProxyMiddleware(object):
@classmethod
def from_crawler(cls, crawler):
+ if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
+ raise NotConfigured
auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
return cls(auth_encoding)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index 24714a7a8..cb88bc2bf 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -174,6 +174,7 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
HTTPCACHE_GZIP = False
+HTTPPROXY_ENABLED = True
HTTPPROXY_AUTH_ENCODING = 'latin-1'
IMAGES_STORE_S3_ACL = 'private'
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index dd09e4dd0..c77179ceb 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -1,11 +1,14 @@
import os
import sys
+from functools import partial
from twisted.trial.unittest import TestCase, SkipTest
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
from scrapy.exceptions import NotConfigured
from scrapy.http import Response, Request
from scrapy.spiders import Spider
+from scrapy.crawler import Crawler
+from scrapy.settings import Settings
spider = Spider('foo')
@@ -20,6 +23,11 @@ class TestDefaultHeadersMiddleware(TestCase):
def tearDown(self):
os.environ = self._oldenv
+ def test_not_enabled(self):
+ settings = Settings({'HTTPPROXY_ENABLED': False})
+ crawler = Crawler(spider, settings)
+ self.assertRaises(NotConfigured, partial(HttpProxyMiddleware.from_crawler, crawler))
+
def test_no_enviroment_proxies(self):
os.environ = {'dummy_proxy': 'reset_env_and_do_not_raise'}
mw = HttpProxyMiddleware()