Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

adding configurable encoding for httpproxy authentication

Raul Gallegosa06a5f00f4a6
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rst
index 3641da231..a97d5a696 100644
--- a/docs/topics/downloader-middleware.rst
+++ b/docs/topics/downloader-middleware.rst
@@ -951,6 +951,18 @@ Default: ``False``
Whether the AjaxCrawlMiddleware will be enabled. You may want to
enable it for :ref:`broad crawls <topics-broad-crawls>`.
+HttpProxyMiddleware settings
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. setting:: HTTPPROXY_AUTH_ENCODING
+
+HTTPPROXY_AUTH_ENCODING
+^^^^^^^^^^^^^^^^^^^^^^^
+
+Default: ``"latin-1"``
+
+The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.
+
.. _DBM: http://en.wikipedia.org/wiki/Dbm
.. _anydbm: https://docs.python.org/2/library/anydbm.html
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 8c3514fd0..b01bab76d 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cached
from scrapy.exceptions import NotConfigured
from scrapy.utils.python import to_bytes
+
class HttpProxyMiddleware(object):
- def __init__(self):
+ def __init__(self, auth_encoding='latin-1'):
+ self.auth_encoding = auth_encoding
self.proxies = {}
for type, url in getproxies().items():
self.proxies[type] = self._get_proxy(url, type)
@@ -21,12 +23,19 @@ class HttpProxyMiddleware(object):
if not self.proxies:
raise NotConfigured
+ @classmethod
+ def from_crawler(cls, crawler):
+ auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
+ return cls(auth_encoding)
+
def _get_proxy(self, url, orig_type):
proxy_type, user, password, hostport = _parse_proxy(url)
proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
if user:
- user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)))
+ user_pass = to_bytes(
+ '%s:%s' % (unquote(user), unquote(password)),
+ encoding=self.auth_encoding)
creds = base64.b64encode(user_pass).strip()
else:
creds = None
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
index b151933b6..44e74dc61 100644
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
HTTPCACHE_GZIP = False
+HTTPPROXY_AUTH_ENCODING = 'latin-1'
+
ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
ITEM_PIPELINES = {}
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 7676b2a00..2b26431a4 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -9,6 +9,7 @@ from scrapy.spiders import Spider
spider = Spider('foo')
+
class TestDefaultHeadersMiddleware(TestCase):
failureException = AssertionError
@@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase):
self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')
+ def test_proxy_auth_encoding(self):
+ # utf-8 encoding
+ os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'
+ mw = HttpProxyMiddleware(auth_encoding='utf-8')
+ req = Request('http://scrapytest.org')
+ assert mw.process_request(req, spider) is None
+ self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+ self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')
+
+ # default latin-1 encoding
+ mw = HttpProxyMiddleware(auth_encoding='latin-1')
+ req = Request('http://scrapytest.org')
+ assert mw.process_request(req, spider) is None
+ self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})
+ self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')
+
def test_proxy_already_seted(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
@@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase):
assert mw.process_request(req, spider) is None
assert 'proxy' in req.meta and req.meta['proxy'] is None
-
def test_no_proxy(self):
os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
mw = HttpProxyMiddleware()
@@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase):
req = Request('http://noproxy.com')
assert mw.process_request(req, spider) is None
assert 'proxy' not in req.meta
-