Scrapy, a fast high-level web crawling & scraping framework for Python.
Relax Proxy-Authorization restrictions
Adrián Chaves77c055ee28a7
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.pyindex 1deda42bd..dd8a7e797 100644--- a/scrapy/downloadermiddlewares/httpproxy.py+++ b/scrapy/downloadermiddlewares/httpproxy.py@@ -78,4 +78,7 @@ class HttpProxyMiddleware:del request.headers[b'Proxy-Authorization']del request.meta['_auth_proxy']elif b'Proxy-Authorization' in request.headers:- del request.headers[b'Proxy-Authorization']+ if proxy_url:+ request.meta['_auth_proxy'] = proxy_url+ else:+ del request.headers[b'Proxy-Authorization']diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.pyindex 70eb94d77..44434f90e 100644--- a/tests/test_downloadermiddleware_httpproxy.py+++ b/tests/test_downloadermiddleware_httpproxy.py@@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase):self.assertNotIn(b'Proxy-Authorization', request.headers)def test_proxy_authentication_header_proxy_without_credentials(self):+ """As long as the proxy URL in request metadata remains the same, the+ Proxy-Authorization header is used and kept, and may even be+ changed."""middleware = HttpProxyMiddleware()request = Request('https://example.com',@@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase):)assert middleware.process_request(request, spider) is Noneself.assertEqual(request.meta['proxy'], 'https://example.com')- self.assertNotIn(b'Proxy-Authorization', request.headers)+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')++ assert middleware.process_request(request, spider) is None+ self.assertEqual(request.meta['proxy'], 'https://example.com')+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')++ request.headers['Proxy-Authorization'] = b'Basic bar'+ assert middleware.process_request(request, spider) is None+ self.assertEqual(request.meta['proxy'], 'https://example.com')+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')def test_proxy_authentication_header_proxy_with_same_credentials(self):middleware = HttpProxyMiddleware()