Skip to content

Scrapy, a fast high-level web crawling & scraping framework for Python.

Relax Proxy-Authorization restrictions

Adrián Chaves77c055ee28a7
diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
index 1deda42bd..dd8a7e797 100644
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -78,4 +78,7 @@ class HttpProxyMiddleware:
del request.headers[b'Proxy-Authorization']
del request.meta['_auth_proxy']
elif b'Proxy-Authorization' in request.headers:
- del request.headers[b'Proxy-Authorization']
+ if proxy_url:
+ request.meta['_auth_proxy'] = proxy_url
+ else:
+ del request.headers[b'Proxy-Authorization']
diff --git a/tests/test_downloadermiddleware_httpproxy.py b/tests/test_downloadermiddleware_httpproxy.py
index 70eb94d77..44434f90e 100644
--- a/tests/test_downloadermiddleware_httpproxy.py
+++ b/tests/test_downloadermiddleware_httpproxy.py
@@ -400,6 +400,9 @@ class TestHttpProxyMiddleware(TestCase):
self.assertNotIn(b'Proxy-Authorization', request.headers)
def test_proxy_authentication_header_proxy_without_credentials(self):
+ """As long as the proxy URL in request metadata remains the same, the
+ Proxy-Authorization header is used and kept, and may even be
+ changed."""
middleware = HttpProxyMiddleware()
request = Request(
'https://example.com',
@@ -408,7 +411,16 @@ class TestHttpProxyMiddleware(TestCase):
)
assert middleware.process_request(request, spider) is None
self.assertEqual(request.meta['proxy'], 'https://example.com')
- self.assertNotIn(b'Proxy-Authorization', request.headers)
+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+ assert middleware.process_request(request, spider) is None
+ self.assertEqual(request.meta['proxy'], 'https://example.com')
+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic foo')
+
+ request.headers['Proxy-Authorization'] = b'Basic bar'
+ assert middleware.process_request(request, spider) is None
+ self.assertEqual(request.meta['proxy'], 'https://example.com')
+ self.assertEqual(request.headers['Proxy-Authorization'], b'Basic bar')
def test_proxy_authentication_header_proxy_with_same_credentials(self):
middleware = HttpProxyMiddleware()