Scrapy, a fast high-level web crawling & scraping framework for Python.
adding configurable encoding for httpproxy authentication
Raul Gallegosa06a5f00f4a6
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 3641da231..a97d5a696 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -951,6 +951,18 @@ Default: ``False``Whether the AjaxCrawlMiddleware will be enabled. You may want toenable it for :ref:`broad crawls <topics-broad-crawls>`.+HttpProxyMiddleware settings+~~~~~~~~~~~~~~~~~~~~~~~~~~~~++.. setting:: HTTPPROXY_AUTH_ENCODING++HTTPPROXY_AUTH_ENCODING+^^^^^^^^^^^^^^^^^^^^^^^++Default: ``"latin-1"``++The default encoding for proxy authentication on :class:`HttpProxyMiddleware`.+.. _DBM: http://en.wikipedia.org/wiki/Dbm.. _anydbm: https://docs.python.org/2/library/anydbm.htmldiff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex 8c3514fd0..b01bab76d 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -11,9 +11,11 @@ from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.exceptions import NotConfiguredfrom scrapy.utils.python import to_bytes+class HttpProxyMiddleware(object):- def __init__(self):+ def __init__(self, auth_encoding='latin-1'):+ self.auth_encoding = auth_encodingself.proxies = {}for type, url in getproxies().items():self.proxies[type] = self._get_proxy(url, type)@@ -21,12 +23,19 @@ class HttpProxyMiddleware(object):if not self.proxies:raise NotConfigured+ @classmethod+ def from_crawler(cls, crawler):+ auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')+ return cls(auth_encoding)+def _get_proxy(self, url, orig_type):proxy_type, user, password, hostport = _parse_proxy(url)proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))if user:- user_pass = to_bytes('%s:%s' % (unquote(user), unquote(password)))+ user_pass = to_bytes(+ '%s:%s' % (unquote(user), unquote(password)),+ encoding=self.auth_encoding)creds = base64.b64encode(user_pass).strip()else:creds = Nonediff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.pyindex b151933b6..44e74dc61 100644--- a/scrapy/settings/default_settings.py+++ b/scrapy/settings/default_settings.py@@ -169,6 +169,8 @@ HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'HTTPCACHE_GZIP = False+HTTPPROXY_AUTH_ENCODING = 'latin-1'+ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'ITEM_PIPELINES = {}diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 7676b2a00..2b26431a4 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -9,6 +9,7 @@ from scrapy.spiders import Spiderspider = Spider('foo')+class TestDefaultHeadersMiddleware(TestCase):failureException = AssertionError@@ -62,6 +63,22 @@ class TestDefaultHeadersMiddleware(TestCase):self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic dXNlcjo=')+ def test_proxy_auth_encoding(self):+ # utf-8 encoding+ os.environ['http_proxy'] = u'https://m\u00E1n:pass@proxy:3128'+ mw = HttpProxyMiddleware(auth_encoding='utf-8')+ req = Request('http://scrapytest.org')+ assert mw.process_request(req, spider) is None+ self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})+ self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic bcOhbjpwYXNz')++ # default latin-1 encoding+ mw = HttpProxyMiddleware(auth_encoding='latin-1')+ req = Request('http://scrapytest.org')+ assert mw.process_request(req, spider) is None+ self.assertEquals(req.meta, {'proxy': 'https://proxy:3128'})+ self.assertEquals(req.headers.get('Proxy-Authorization'), b'Basic beFuOnBhc3M=')+def test_proxy_already_seted(self):os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'mw = HttpProxyMiddleware()@@ -69,7 +86,6 @@ class TestDefaultHeadersMiddleware(TestCase):assert mw.process_request(req, spider) is Noneassert 'proxy' in req.meta and req.meta['proxy'] is None-def test_no_proxy(self):os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'mw = HttpProxyMiddleware()@@ -88,4 +104,3 @@ class TestDefaultHeadersMiddleware(TestCase):req = Request('http://noproxy.com')assert mw.process_request(req, spider) is Noneassert 'proxy' not in req.meta-