Scrapy, a fast high-level web crawling & scraping framework for Python.
Add http_auth_domain to HttpAuthMiddleware.
Andrey Rakhmatullinb081f18a2f23
diff --git a/docs/topics/downloader-middleware.rst b/docs/topics/downloader-middleware.rstindex 99d57bda9..28b019c80 100644--- a/docs/topics/downloader-middleware.rst+++ b/docs/topics/downloader-middleware.rst@@ -323,8 +323,21 @@ HttpAuthMiddlewareThis middleware authenticates all requests generated from certain spidersusing `Basic access authentication`_ (aka. HTTP auth).- To enable HTTP authentication from certain spiders, set the ``http_user``- and ``http_pass`` attributes of those spiders.+ To enable HTTP authentication for a spider, set the ``http_user`` and+ ``http_pass`` spider attributes to the authentication data and the+ ``http_auth_domain`` spider attribute to the domain which requires this+ authentication (its subdomains will be also handled in the same way).+ You can set ``http_auth_domain`` to ``None`` to enable the+ authentication for all requests but usually this is not needed.++ .. warning::+ In the previous Scrapy versions HttpAuthMiddleware sent the+ authentication data with all requests, which is a security problem if+ the spider makes requests to several different domains. Currently if+ the ``http_auth_domain`` attribute is not set, the middleware will use+ the domain of the first request, which will work for some spider but+ not for others. In the future the middleware will produce an error+ instead.Example::@@ -334,6 +347,7 @@ HttpAuthMiddlewarehttp_user = 'someuser'http_pass = 'somepass'+ http_auth_domain = 'intranet.example.com'name = 'intranet.example.com'# .. rest of the spider code omitted ...diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex 089bf0d85..1bee3e279 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -3,10 +3,14 @@ HTTP basic auth downloader middlewareSee documentation in docs/topics/downloader-middleware.rst"""+import warningsfrom w3lib.http import basic_auth_headerfrom scrapy import signals+from scrapy.exceptions import ScrapyDeprecationWarning+from scrapy.utils.httpobj import urlparse_cached+from scrapy.utils.url import url_is_from_any_domainclass HttpAuthMiddleware:@@ -24,8 +28,23 @@ class HttpAuthMiddleware:pwd = getattr(spider, 'http_pass', '')if usr or pwd:self.auth = basic_auth_header(usr, pwd)+ if not hasattr(spider, 'http_auth_domain'):+ warnings.warn('Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security '+ 'problems if the spider makes requests to several different domains. http_auth_domain '+ 'will be set to the domain of the first request, please set it to the correct value '+ 'explicitly.',+ category=ScrapyDeprecationWarning)+ self.domain_unset = True+ else:+ self.domain = spider.http_auth_domain+ self.domain_unset = Falsedef process_request(self, request, spider):auth = getattr(self, 'auth', None)if auth and b'Authorization' not in request.headers:- request.headers[b'Authorization'] = auth+ domain = urlparse_cached(request).hostname+ if self.domain_unset:+ self.domain = domain+ self.domain_unset = False+ if not self.domain or url_is_from_any_domain(request.url, [self.domain]):+ request.headers[b'Authorization'] = authdiff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 3381632b0..0362e2018 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -1,13 +1,60 @@import unittest+from w3lib.http import basic_auth_header+from scrapy.http import Requestfrom scrapy.downloadermiddlewares.httpauth import HttpAuthMiddlewarefrom scrapy.spiders import Spider+class TestSpiderLegacy(Spider):+ http_user = 'foo'+ http_pass = 'bar'++class TestSpider(Spider):http_user = 'foo'http_pass = 'bar'+ http_auth_domain = 'example.com'+++class TestSpiderAny(Spider):+ http_user = 'foo'+ http_pass = 'bar'+ http_auth_domain = None+++class HttpAuthMiddlewareLegacyTest(unittest.TestCase):++ def setUp(self):+ self.spider = TestSpiderLegacy('foo')++ def test_auth(self):+ mw = HttpAuthMiddleware()+ mw.spider_opened(self.spider)++ # initial request, sets the domain and sends the header+ req = Request('http://example.com/')+ assert mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))++ # subsequent request to the same domain, should send the header+ req = Request('http://example.com/')+ assert mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))++ # subsequent request to a different domain, shouldn't send the header+ req = Request('http://example-noauth.com/')+ assert mw.process_request(req, self.spider) is None+ self.assertNotIn('Authorization', req.headers)++ def test_auth_already_set(self):+ mw = HttpAuthMiddleware()+ mw.spider_opened(self.spider)+ req = Request('http://example.com/',+ headers=dict(Authorization='Digest 123'))+ assert mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], b'Digest 123')class HttpAuthMiddlewareTest(unittest.TestCase):@@ -20,13 +67,45 @@ class HttpAuthMiddlewareTest(unittest.TestCase):def tearDown(self):del self.mw+ def test_no_auth(self):+ req = Request('http://example-noauth.com/')+ assert self.mw.process_request(req, self.spider) is None+ self.assertNotIn('Authorization', req.headers)++ def test_auth_domain(self):+ req = Request('http://example.com/')+ assert self.mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))++ def test_auth_subdomain(self):+ req = Request('http://foo.example.com/')+ assert self.mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))++ def test_auth_already_set(self):+ req = Request('http://example.com/',+ headers=dict(Authorization='Digest 123'))+ assert self.mw.process_request(req, self.spider) is None+ self.assertEqual(req.headers['Authorization'], b'Digest 123')+++class HttpAuthAnyMiddlewareTest(unittest.TestCase):++ def setUp(self):+ self.mw = HttpAuthMiddleware()+ self.spider = TestSpiderAny('foo')+ self.mw.spider_opened(self.spider)++ def tearDown(self):+ del self.mw+def test_auth(self):- req = Request('http://scrapytest.org/')+ req = Request('http://example.com/')assert self.mw.process_request(req, self.spider) is None- self.assertEqual(req.headers['Authorization'], b'Basic Zm9vOmJhcg==')+ self.assertEqual(req.headers['Authorization'], basic_auth_header('foo', 'bar'))def test_auth_already_set(self):- req = Request('http://scrapytest.org/',+ req = Request('http://example.com/',headers=dict(Authorization='Digest 123'))assert self.mw.process_request(req, self.spider) is Noneself.assertEqual(req.headers['Authorization'], b'Digest 123')