Scrapy, a fast high-level web crawling & scraping framework for Python.
Remove support for using HttpAuthMiddleware without http_auth_domain
Laerte Pereiraa2b9351f04f0
diff --git a/scrapy/downloadermiddlewares/httpauth.py b/scrapy/downloadermiddlewares/httpauth.pyindex de5a81388..5228db786 100644--- a/scrapy/downloadermiddlewares/httpauth.py+++ b/scrapy/downloadermiddlewares/httpauth.py@@ -3,13 +3,10 @@ HTTP basic auth downloader middlewareSee documentation in docs/topics/downloader-middleware.rst"""-import warningsfrom w3lib.http import basic_auth_headerfrom scrapy import signals-from scrapy.exceptions import ScrapyDeprecationWarning-from scrapy.utils.httpobj import urlparse_cachedfrom scrapy.utils.url import url_is_from_any_domain@@ -28,25 +25,10 @@ class HttpAuthMiddleware:pwd = getattr(spider, "http_pass", "")if usr or pwd:self.auth = basic_auth_header(usr, pwd)- if not hasattr(spider, "http_auth_domain"):- warnings.warn(- "Using HttpAuthMiddleware without http_auth_domain is deprecated and can cause security "- "problems if the spider makes requests to several different domains. http_auth_domain "- "will be set to the domain of the first request, please set it to the correct value "- "explicitly.",- category=ScrapyDeprecationWarning,- )- self.domain_unset = True- else:- self.domain = spider.http_auth_domain- self.domain_unset = False+ self.domain = spider.http_auth_domaindef process_request(self, request, spider):auth = getattr(self, "auth", None)if auth and b"Authorization" not in request.headers:- domain = urlparse_cached(request).hostname- if self.domain_unset:- self.domain = domain- self.domain_unset = Falseif not self.domain or url_is_from_any_domain(request.url, [self.domain]):request.headers[b"Authorization"] = authdiff --git a/tests/test_downloadermiddleware_httpauth.py b/tests/test_downloadermiddleware_httpauth.pyindex 6b79234d0..fc110e6cc 100644--- a/tests/test_downloadermiddleware_httpauth.py+++ b/tests/test_downloadermiddleware_httpauth.py@@ -1,10 +1,8 @@import unittest-import pytestfrom w3lib.http import basic_auth_headerfrom scrapy.downloadermiddlewares.httpauth import HttpAuthMiddleware-from scrapy.exceptions import ScrapyDeprecationWarningfrom scrapy.http import Requestfrom scrapy.spiders import Spider@@ -31,39 +29,10 @@ class HttpAuthMiddlewareLegacyTest(unittest.TestCase):self.spider = TestSpiderLegacy("foo")def test_auth(self):- with pytest.warns(- ScrapyDeprecationWarning,- match="Using HttpAuthMiddleware without http_auth_domain is deprecated",- ):+ with self.assertRaises(AttributeError):mw = HttpAuthMiddleware()mw.spider_opened(self.spider)- # initial request, sets the domain and sends the header- req = Request("http://example.com/")- assert mw.process_request(req, self.spider) is None- self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))-- # subsequent request to the same domain, should send the header- req = Request("http://example.com/")- assert mw.process_request(req, self.spider) is None- self.assertEqual(req.headers["Authorization"], basic_auth_header("foo", "bar"))-- # subsequent request to a different domain, shouldn't send the header- req = Request("http://example-noauth.com/")- assert mw.process_request(req, self.spider) is None- self.assertNotIn("Authorization", req.headers)-- def test_auth_already_set(self):- with pytest.warns(- ScrapyDeprecationWarning,- match="Using HttpAuthMiddleware without http_auth_domain is deprecated",- ):- mw = HttpAuthMiddleware()- mw.spider_opened(self.spider)- req = Request("http://example.com/", headers=dict(Authorization="Digest 123"))- assert mw.process_request(req, self.spider) is None- self.assertEqual(req.headers["Authorization"], b"Digest 123")-class HttpAuthMiddlewareTest(unittest.TestCase):def setUp(self):